mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
feat: add S3-compatible storage service
This commit is contained in:
1 parent
45a0168a77
commit
9faf165613
17 files changed
+3425
-1
No files matched your search
@@ -621,11 +621,29 @@ return function (Container $context): void {
|
||||
$projectId = (string) $request->getQuery('project', '');
|
||||
}
|
||||
|
||||
$route = $utopia->match($request)?->route;
|
||||
|
||||
// S3 uses the Appwrite project ID as its SigV4 access key. Header-signed
|
||||
// requests carry the credential scope in Authorization; presigned URLs
|
||||
// carry it in the X-Amz-Credential query parameter.
|
||||
if ($projectId === '' && \in_array('s3', $route?->getGroups() ?? [], true)) {
|
||||
$credential = '';
|
||||
$authorizationHeader = $request->getHeaderLine('authorization', '');
|
||||
if (\preg_match('/Credential=([^\s,]+)/', $authorizationHeader, $matches) === 1) {
|
||||
$credential = $matches[1];
|
||||
} else {
|
||||
$credential = $request->getQuery('X-Amz-Credential', '');
|
||||
}
|
||||
|
||||
if (\is_string($credential)) {
|
||||
$projectId = \explode('/', $credential, 2)[0];
|
||||
}
|
||||
}
|
||||
|
||||
// Backwards compatibility for new services, originally project resources
|
||||
// These endpoints moved from /v1/projects/:projectId/<resource> to /v1/<resource>
|
||||
// When accessed via the old alias path, extract projectId from the URI
|
||||
$deprecatedProjectPathPrefix = '/v1/projects/';
|
||||
$route = $utopia->match($request)?->route;
|
||||
if (!empty($route)) {
|
||||
$isDeprecatedAlias = $projectIdFromPath !== '' &&
|
||||
!\str_starts_with($route->getPath(), $deprecatedProjectPathPrefix);
|
||||
|
||||
@@ -17,6 +17,7 @@ use Appwrite\Platform\Modules\Presences;
|
||||
use Appwrite\Platform\Modules\Project;
|
||||
use Appwrite\Platform\Modules\Projects;
|
||||
use Appwrite\Platform\Modules\Proxy;
|
||||
use Appwrite\Platform\Modules\S3;
|
||||
use Appwrite\Platform\Modules\Sites;
|
||||
use Appwrite\Platform\Modules\Storage;
|
||||
use Appwrite\Platform\Modules\Teams;
|
||||
@@ -43,6 +44,7 @@ class Appwrite extends Platform
|
||||
$this->addModule(new Sites\Module());
|
||||
$this->addModule(new Console\Module());
|
||||
$this->addModule(new Proxy\Module());
|
||||
$this->addModule(new S3\Module());
|
||||
$this->addModule(new Teams\Module());
|
||||
$this->addModule(new Tokens\Module());
|
||||
$this->addModule(new Users\Module());
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Auth;
|
||||
|
||||
use Appwrite\Auth\Key;
|
||||
use Appwrite\Extend\Exception;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
|
||||
class SignatureV4
|
||||
{
|
||||
private const MAX_CLOCK_SKEW = 900;
|
||||
|
||||
private const MAX_PRESIGNED_EXPIRY = 604800;
|
||||
|
||||
/**
|
||||
* The AWS access key ID is the Appwrite project ID; the AWS secret is an Appwrite API key secret.
|
||||
*/
|
||||
public function verify(Request $request, Document $project, Document $team, User $user, array $requiredScopes): Key
|
||||
{
|
||||
$authorization = $request->getHeaderLine('authorization');
|
||||
$queryAuthentication = $this->queryAuthentication($request);
|
||||
|
||||
if ($authorization !== '' && $queryAuthentication !== null) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Multiple AWS Signature V4 authentication methods were provided.');
|
||||
}
|
||||
|
||||
if ($queryAuthentication !== null) {
|
||||
$algorithm = $queryAuthentication['X-Amz-Algorithm'] ?? '';
|
||||
if ($algorithm !== 'AWS4-HMAC-SHA256') {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Unsupported AWS Signature V4 algorithm.');
|
||||
}
|
||||
|
||||
$credential = \explode('/', $queryAuthentication['X-Amz-Credential'] ?? '');
|
||||
$signedHeaders = \explode(';', $queryAuthentication['X-Amz-SignedHeaders'] ?? '');
|
||||
$signature = $queryAuthentication['X-Amz-Signature'] ?? '';
|
||||
$date = $queryAuthentication['X-Amz-Date'] ?? '';
|
||||
$expires = $queryAuthentication['X-Amz-Expires'] ?? '';
|
||||
$presigned = true;
|
||||
} else {
|
||||
if (!\str_starts_with($authorization, 'AWS4-HMAC-SHA256 ')) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Missing AWS Signature V4 authentication.');
|
||||
}
|
||||
|
||||
$parts = $this->parseAuthorization($authorization);
|
||||
$credential = \explode('/', $parts['Credential'] ?? '');
|
||||
$signedHeaders = \explode(';', $parts['SignedHeaders'] ?? '');
|
||||
$signature = $parts['Signature'] ?? '';
|
||||
$date = $request->getHeaderLine('x-amz-date') ?: $request->getHeaderLine('date');
|
||||
$expires = null;
|
||||
$presigned = false;
|
||||
}
|
||||
|
||||
if (
|
||||
\count($credential) !== 5
|
||||
|| $credential[1] === ''
|
||||
|| $credential[2] === ''
|
||||
|| $credential[3] !== 's3'
|
||||
|| $credential[4] !== 'aws4_request'
|
||||
) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 credential scope.');
|
||||
}
|
||||
|
||||
$projectId = $credential[0];
|
||||
|
||||
if ($projectId === '') {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Missing AWS credential project ID.');
|
||||
}
|
||||
|
||||
if ($projectId !== $project->getId()) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS credential project ID does not belong to the selected project.');
|
||||
}
|
||||
|
||||
$normalizedSignedHeaders = \array_values(\array_filter(\array_map(
|
||||
fn (string $header): string => \strtolower(\trim($header)),
|
||||
$signedHeaders
|
||||
)));
|
||||
$sortedSignedHeaders = $normalizedSignedHeaders;
|
||||
\sort($sortedSignedHeaders, SORT_STRING);
|
||||
if (
|
||||
$signature === ''
|
||||
|| $date === ''
|
||||
|| ($presigned && \preg_match('/^[a-f0-9]{64}$/D', $signature) !== 1)
|
||||
|| ($presigned && !\in_array('host', $normalizedSignedHeaders, true))
|
||||
|| ($presigned && $normalizedSignedHeaders !== $sortedSignedHeaders)
|
||||
) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Incomplete AWS Signature V4 authentication.');
|
||||
}
|
||||
|
||||
if ($presigned) {
|
||||
$this->validatePresignedDate($date, $credential[1], (string) $expires);
|
||||
} else {
|
||||
$this->validateDate($date, $credential[1]);
|
||||
}
|
||||
|
||||
$canonicalRequest = $this->canonicalRequest($request, $normalizedSignedHeaders, $presigned);
|
||||
$scope = \implode('/', \array_slice($credential, 1, 4));
|
||||
$stringToSign = "AWS4-HMAC-SHA256\n{$date}\n{$scope}\n" . \hash('sha256', $canonicalRequest);
|
||||
|
||||
foreach ($this->keySecrets($project) as $secret) {
|
||||
$signingKey = $this->signingKey($secret, $credential[1], $credential[2], $credential[3]);
|
||||
$expected = \hash_hmac('sha256', $stringToSign, $signingKey);
|
||||
|
||||
if (!\hash_equals($expected, $signature)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$apiKey = Key::decode($project, $team, $user, $secret);
|
||||
if ($apiKey->getRole() !== User::ROLE_KEYS || $apiKey->isExpired()) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid or expired API key.');
|
||||
}
|
||||
|
||||
if (empty(\array_intersect($requiredScopes, $apiKey->getScopes()))) {
|
||||
throw new Exception(Exception::GENERAL_UNAUTHORIZED_SCOPE, 'API key is missing required S3 storage scopes.');
|
||||
}
|
||||
|
||||
return $apiKey;
|
||||
}
|
||||
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Signature does not match.');
|
||||
}
|
||||
|
||||
private function validatePresignedDate(string $date, string $credentialDate, string $expires): void
|
||||
{
|
||||
if ($expires === '' || !\ctype_digit($expires)) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 presigned URL expiry.');
|
||||
}
|
||||
|
||||
$expires = (int) $expires;
|
||||
if ($expires < 1 || $expires > self::MAX_PRESIGNED_EXPIRY) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 presigned URL expiry is outside the allowed range.');
|
||||
}
|
||||
|
||||
$timestamp = $this->timestamp($date);
|
||||
if ($timestamp->format('Ymd') !== $credentialDate) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 credential date does not match request timestamp.');
|
||||
}
|
||||
|
||||
$now = \time();
|
||||
if ($timestamp->getTimestamp() - $now > self::MAX_CLOCK_SKEW) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 timestamp is outside the allowed time window.');
|
||||
}
|
||||
if ($now > $timestamp->getTimestamp() + $expires) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 presigned URL has expired.');
|
||||
}
|
||||
}
|
||||
|
||||
private function validateDate(string $date, string $credentialDate): void
|
||||
{
|
||||
$timestamp = \DateTimeImmutable::createFromFormat('!Ymd\THis\Z', $date, new \DateTimeZone('UTC'));
|
||||
if ($timestamp === false) {
|
||||
$parsed = \strtotime($date);
|
||||
if ($parsed === false) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 timestamp.');
|
||||
}
|
||||
|
||||
$timestamp = (new \DateTimeImmutable('@' . $parsed))->setTimezone(new \DateTimeZone('UTC'));
|
||||
}
|
||||
|
||||
if ($timestamp->format('Ymd') !== $credentialDate) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 credential date does not match request timestamp.');
|
||||
}
|
||||
|
||||
if (\abs(\time() - $timestamp->getTimestamp()) > self::MAX_CLOCK_SKEW) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'AWS Signature V4 timestamp is outside the allowed time window.');
|
||||
}
|
||||
}
|
||||
|
||||
private function timestamp(string $date): \DateTimeImmutable
|
||||
{
|
||||
$timestamp = \DateTimeImmutable::createFromFormat('!Ymd\THis\Z', $date, new \DateTimeZone('UTC'));
|
||||
$errors = \DateTimeImmutable::getLastErrors();
|
||||
if (
|
||||
$timestamp === false
|
||||
|| (\is_array($errors) && ($errors['warning_count'] > 0 || $errors['error_count'] > 0))
|
||||
|| $timestamp->format('Ymd\THis\Z') !== $date
|
||||
) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Invalid AWS Signature V4 timestamp.');
|
||||
}
|
||||
|
||||
return $timestamp;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string>
|
||||
*/
|
||||
private function keySecrets(Document $project): array
|
||||
{
|
||||
$secrets = [];
|
||||
foreach ($project->getAttribute('keys', []) as $key) {
|
||||
$secret = $key instanceof Document
|
||||
? $key->getAttribute('secret', '')
|
||||
: (string) ($key['secret'] ?? '');
|
||||
|
||||
if ($secret !== '') {
|
||||
$secrets[] = $secret;
|
||||
}
|
||||
}
|
||||
|
||||
return $secrets;
|
||||
}
|
||||
|
||||
private function parseAuthorization(string $authorization): array
|
||||
{
|
||||
$authorization = \substr($authorization, \strlen('AWS4-HMAC-SHA256 '));
|
||||
$result = [];
|
||||
foreach (\explode(',', $authorization) as $part) {
|
||||
[$key, $value] = \array_pad(\explode('=', \trim($part), 2), 2, '');
|
||||
$result[$key] = $value;
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, string>|null
|
||||
*/
|
||||
private function queryAuthentication(Request $request): ?array
|
||||
{
|
||||
$authenticationParameters = [
|
||||
'X-Amz-Algorithm',
|
||||
'X-Amz-Credential',
|
||||
'X-Amz-Date',
|
||||
'X-Amz-Expires',
|
||||
'X-Amz-Signature',
|
||||
'X-Amz-SignedHeaders',
|
||||
];
|
||||
$parameters = [];
|
||||
$hasAuthenticationParameter = false;
|
||||
|
||||
foreach ($this->queryPairs($request) as [$name, $value]) {
|
||||
if (!\in_array($name, $authenticationParameters, true)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$hasAuthenticationParameter = true;
|
||||
if (\array_key_exists($name, $parameters)) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Duplicate AWS Signature V4 authentication parameter.');
|
||||
}
|
||||
$parameters[$name] = $value;
|
||||
}
|
||||
|
||||
if (!$hasAuthenticationParameter) {
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach ($authenticationParameters as $parameter) {
|
||||
if (!\array_key_exists($parameter, $parameters)) {
|
||||
throw new Exception(Exception::USER_UNAUTHORIZED, 'Incomplete AWS Signature V4 query authentication.');
|
||||
}
|
||||
}
|
||||
|
||||
return $parameters;
|
||||
}
|
||||
|
||||
private function canonicalRequest(Request $request, array $signedHeaders, bool $presigned): string
|
||||
{
|
||||
$headers = [];
|
||||
foreach ($signedHeaders as $header) {
|
||||
$header = \strtolower(\trim($header));
|
||||
if ($header === '') {
|
||||
continue;
|
||||
}
|
||||
$headers[$header] = \preg_replace('/\s+/', ' ', \trim($request->getHeaderLine($header)));
|
||||
}
|
||||
\ksort($headers);
|
||||
|
||||
$canonicalHeaders = '';
|
||||
foreach ($headers as $key => $value) {
|
||||
$canonicalHeaders .= $key . ':' . $value . "\n";
|
||||
}
|
||||
|
||||
$payloadHash = $presigned ? 'UNSIGNED-PAYLOAD' : $request->getHeaderLine('x-amz-content-sha256');
|
||||
$payloadHash = $payloadHash === '' ? \hash('sha256', $request->getRawPayload()) : $payloadHash;
|
||||
|
||||
return \implode("\n", [
|
||||
$request->getMethod(),
|
||||
$this->canonicalUri($request),
|
||||
$this->canonicalQuery($request, $presigned),
|
||||
$canonicalHeaders,
|
||||
\implode(';', \array_keys($headers)),
|
||||
$payloadHash,
|
||||
]);
|
||||
}
|
||||
|
||||
private function canonicalUri(Request $request): string
|
||||
{
|
||||
$path = \parse_url($request->getURI(), PHP_URL_PATH);
|
||||
return $path === null || $path === false || $path === '' ? '/' : $path;
|
||||
}
|
||||
|
||||
private function canonicalQuery(Request $request, bool $presigned): string
|
||||
{
|
||||
$pairs = [];
|
||||
foreach ($this->queryPairs($request) as [$key, $value]) {
|
||||
if ($presigned && $key === 'X-Amz-Signature') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$pairs[] = [\rawurlencode($key), \rawurlencode($value)];
|
||||
}
|
||||
|
||||
\usort($pairs, function (array $left, array $right): int {
|
||||
$keyOrder = \strcmp($left[0], $right[0]);
|
||||
return $keyOrder === 0 ? \strcmp($left[1], $right[1]) : $keyOrder;
|
||||
});
|
||||
|
||||
return \implode('&', \array_map(fn (array $pair): string => $pair[0] . '=' . $pair[1], $pairs));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array{0: string, 1: string}>
|
||||
*/
|
||||
private function queryPairs(Request $request): array
|
||||
{
|
||||
$query = $request->getServer('query_string', '') ?: (\parse_url($request->getURI(), PHP_URL_QUERY) ?: '');
|
||||
if ($query === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
$pairs = [];
|
||||
foreach (\explode('&', $query) as $part) {
|
||||
[$name, $value] = \array_pad(\explode('=', $part, 2), 2, '');
|
||||
$pairs[] = [\rawurldecode($name), \rawurldecode($value)];
|
||||
}
|
||||
|
||||
return $pairs;
|
||||
}
|
||||
|
||||
private function signingKey(string $secret, string $date, string $region, string $service): string
|
||||
{
|
||||
$kDate = \hash_hmac('sha256', $date, 'AWS4' . $secret, true);
|
||||
$kRegion = \hash_hmac('sha256', $region, $kDate, true);
|
||||
$kService = \hash_hmac('sha256', $service, $kRegion, true);
|
||||
return \hash_hmac('sha256', 'aws4_request', $kService, true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Http;
|
||||
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Geo\Geo;
|
||||
use Appwrite\Network\Cors;
|
||||
use Appwrite\Usage\Context as UsageContext;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Request;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\System\System;
|
||||
|
||||
/**
|
||||
* The S3 routes run in the `s3` group, not `api`: they authenticate with AWS
|
||||
* Signature V4 instead of Appwrite headers, so none of the `api` hooks fire
|
||||
* for them. The module's hooks carry what the gateway still owes the
|
||||
* platform: the events queue identity here, event fan-out and network
|
||||
* metering in the shutdown hooks. Abuse limits are deliberately absent — the
|
||||
* S3 surface is API-key only.
|
||||
*/
|
||||
class Init extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'init';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->setType(Action::TYPE_INIT)
|
||||
->groups(['s3'])
|
||||
->desc('S3 gateway request setup')
|
||||
->inject('request')
|
||||
->inject('response')
|
||||
->inject('cors')
|
||||
->inject('project')
|
||||
->inject('user')
|
||||
->inject('queueForEvents')
|
||||
->inject('usage')
|
||||
->inject('geo')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(Request $request, Response $response, Cors $cors, Document $project, User $user, Event $queueForEvents, UsageContext $usage, Geo $geo): void
|
||||
{
|
||||
$corsHeaders = $cors->headers($request->getOrigin());
|
||||
$exposedHeaders = \array_filter(\array_map('trim', \explode(',', (string) ($corsHeaders[Cors::HEADER_EXPOSE_HEADERS] ?? ''))));
|
||||
$corsHeaders[Cors::HEADER_EXPOSE_HEADERS] = \implode(', ', \array_unique(\array_merge($exposedHeaders, [
|
||||
'Accept-Ranges',
|
||||
'Content-Length',
|
||||
'Content-Range',
|
||||
'ETag',
|
||||
'Last-Modified',
|
||||
'x-amz-server-side-encryption',
|
||||
])));
|
||||
foreach ($corsHeaders as $name => $value) {
|
||||
$response->addHeader($name, $value);
|
||||
}
|
||||
|
||||
// The S3 actions set the event name and params themselves; project and
|
||||
// user identify the tenant for event fan-out and for the cache-delete
|
||||
// messages the module builds from this queue.
|
||||
$queueForEvents
|
||||
->setProject($project)
|
||||
->setUser($user);
|
||||
|
||||
// Mirrors the `api` group's "Enrich usage context with request
|
||||
// metadata" hook in app/controllers/general.php — the gateway meters
|
||||
// network usage through the same context.
|
||||
$country = '';
|
||||
$geoRecord = $geo->get($request->getIP());
|
||||
if (!$geoRecord->isEmpty()) {
|
||||
$country = \strtolower($geoRecord->getCountryCode());
|
||||
}
|
||||
|
||||
$uri = $request->getURI();
|
||||
$parts = explode('/', trim($uri, '/'));
|
||||
$service = count($parts) >= 2 ? $parts[1] : $parts[0];
|
||||
|
||||
$usage
|
||||
->setPath($uri)
|
||||
->setMethod($request->getMethod())
|
||||
->setUserAgent($request->getUserAgent(''))
|
||||
->setHostname($request->getOrigin('') ?: $request->getHostname())
|
||||
->setCountry($country)
|
||||
->setIp($request->getIP())
|
||||
->setSdk(\strtolower($request->getHeaderLine('x-sdk-name', '')))
|
||||
->setSdkVersion($request->getHeaderLine('x-sdk-version', ''))
|
||||
->setRegion(System::getEnv('_APP_REGION', 'default'))
|
||||
->setService($service);
|
||||
|
||||
if (!$project->isEmpty()) {
|
||||
$usage
|
||||
->setTeamId((string) $project->getAttribute('teamId', ''))
|
||||
->setTeamInternalId((string) $project->getAttribute('teamInternalId', ''));
|
||||
}
|
||||
|
||||
// Always reset — clears values carried over from a previous request
|
||||
// when this Context instance is reused across requests in the same worker.
|
||||
$usage->setResourceType('');
|
||||
$usage->setResourceId('');
|
||||
$usage->setResourceInternalId('');
|
||||
$usage->setResourcePath('');
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,225 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Http\S3;
|
||||
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Audit as AuditPublisher;
|
||||
use Appwrite\Event\Publisher\Delete as DeletePublisher;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Extend\Exception as AppwriteException;
|
||||
use Appwrite\Functions\EventProcessor;
|
||||
use Appwrite\Platform\Modules\S3\Responses\S3Xml;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Storage\Device;
|
||||
|
||||
class Create extends Base
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 's3Create';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct(Action::HTTP_REQUEST_METHOD_PUT);
|
||||
$this->setHttpMethods([Action::HTTP_REQUEST_METHOD_PUT, Action::HTTP_REQUEST_METHOD_POST]);
|
||||
}
|
||||
|
||||
protected function route(Request $request, Response $response, Database $dbForProject, Document $project, Document $team, User $user, Device $deviceForFiles, callable $locks, Cache $cache, Event $queueForEvents, DeletePublisher $publisherForDeletes, AuditPublisher $publisherForAudits, Event $queueForRealtime, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, EventProcessor $eventProcessor): void
|
||||
{
|
||||
[$bucketId, $key] = $this->parts($request);
|
||||
if ($bucketId === '') {
|
||||
throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Bucket is required.');
|
||||
}
|
||||
|
||||
if ($this->unsupportedSubresource($request)) {
|
||||
$this->authorize($request, $project, $team, $user, $key === '' ? ['buckets.write'] : ['files.write']);
|
||||
$this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
|
||||
return;
|
||||
}
|
||||
|
||||
if ($this->hasQuery($request, 'acl')) {
|
||||
$this->authorize($request, $project, $team, $user, $key === '' ? ['buckets.write'] : ['files.write']);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
if ($key !== '') {
|
||||
$this->getObject($dbForProject, $bucket, $key);
|
||||
}
|
||||
$response->setStatusCode(Response::STATUS_CODE_OK)->send('');
|
||||
return;
|
||||
}
|
||||
|
||||
if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_POST && $key === '' && $this->hasQuery($request, 'delete')) {
|
||||
$this->authorize($request, $project, $team, $user, ['files.write']);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
$payload = $this->requestBody($request);
|
||||
$deleted = [];
|
||||
foreach ($this->deleteObjectKeys($payload) as $objectKey) {
|
||||
[$file, $removed] = $locks($this->objectLockKey($project, $bucket, $objectKey), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $objectKey): array {
|
||||
$file = $this->findObject($dbForProject, $bucket, $objectKey);
|
||||
if ($file === null) {
|
||||
return [null, true];
|
||||
}
|
||||
|
||||
$path = $file->getAttribute('path', '');
|
||||
if ($path !== '' && !$deviceForFiles->delete($path)) {
|
||||
// Storage delete failed; keep the DB document so the
|
||||
// object remains reachable and report no deletion.
|
||||
return [$file, false];
|
||||
}
|
||||
$dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $file->getId()));
|
||||
|
||||
return [$file, true];
|
||||
}, timeout: 120.0);
|
||||
|
||||
if (!$removed) {
|
||||
continue;
|
||||
}
|
||||
if ($file !== null) {
|
||||
$this->enqueueFileCacheDelete($publisherForDeletes, $queueForEvents, $bucket, $file);
|
||||
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.delete', $file, 'file');
|
||||
$this->triggerFileEvent($queueForEvents, $queueForRealtime, $publisherForFunctions, $queueForWebhooks, $eventProcessor, $dbForProject, $response, $bucket, $file, 'delete');
|
||||
}
|
||||
$deleted[] = $objectKey;
|
||||
}
|
||||
|
||||
$this->sendXml($response, S3Xml::deleteObjects($deleted, $this->deleteQuiet($payload)));
|
||||
return;
|
||||
}
|
||||
|
||||
if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_POST && $key !== '' && $this->hasQuery($request, 'uploads')) {
|
||||
$this->authorize($request, $project, $team, $user, ['files.write']);
|
||||
if ($this->isFolderMarker($key)) {
|
||||
$this->sendXml($response, S3Xml::error('NotImplemented', 'S3 folders are not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
|
||||
return;
|
||||
}
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
$uploadId = \bin2hex(\random_bytes(16));
|
||||
$this->createMultipartUpload($dbForProject, $cache, $project, $deviceForFiles, $bucket, $key, $uploadId, $request->getHeaderLine('content-type', 'application/octet-stream'), $this->requestObjectMetadata($request));
|
||||
$this->sendXml($response, S3Xml::initiateMultipart($bucketId, $key, $uploadId));
|
||||
return;
|
||||
}
|
||||
|
||||
if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_PUT && $key !== '' && $this->query($request, 'uploadId') !== '' && $this->query($request, 'partNumber') !== '') {
|
||||
$this->authorize($request, $project, $team, $user, ['files.write']);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
$partNumber = (int) $this->query($request, 'partNumber');
|
||||
$uploadId = $this->query($request, 'uploadId');
|
||||
|
||||
$body = $this->requestBody($request);
|
||||
$copySource = $request->getHeaderLine('x-amz-copy-source');
|
||||
if ($copySource !== '') {
|
||||
// Copy reads the source bytes, so the key must also hold files.read.
|
||||
$this->authorize($request, $project, $team, $user, ['files.read']);
|
||||
$copySource = \rawurldecode(\ltrim($copySource, '/'));
|
||||
[$sourceBucketId, $sourceKey] = \array_pad(\explode('/', $copySource, 2), 2, '');
|
||||
$sourceBucket = $this->bucket($dbForProject, $sourceBucketId);
|
||||
$sourceFile = $this->getObject($dbForProject, $sourceBucket, $sourceKey);
|
||||
$this->validateCopySize($bucket, $sourceFile);
|
||||
$body = (string) $deviceForFiles->read($sourceFile->getAttribute('path', ''));
|
||||
if (\preg_match('/^bytes=(\d+)-(\d+)$/', $request->getHeaderLine('x-amz-copy-source-range'), $matches) === 1) {
|
||||
$start = (int) $matches[1];
|
||||
$end = (int) $matches[2];
|
||||
$body = \substr($body, $start, $end - $start + 1);
|
||||
}
|
||||
}
|
||||
|
||||
$etag = \md5($body);
|
||||
// SDKs upload parts of one upload concurrently; the cached parts map
|
||||
// and device chunk bookkeeping are read-modify-write, so load and
|
||||
// save must share the upload's critical section.
|
||||
$locks($this->multipartLockKey($project, $uploadId), 600, function () use ($cache, $project, $deviceForFiles, $uploadId, $bucketId, $key, $partNumber, $body, $etag): void {
|
||||
$upload = $this->multipartUploadForBucket($cache, $project, $uploadId, $bucketId, $key);
|
||||
$this->uploadMultipartPart($cache, $project, $deviceForFiles, $upload, $partNumber, $body, $etag);
|
||||
}, timeout: 120.0);
|
||||
if ($copySource !== '') {
|
||||
$this->sendXml($response, S3Xml::copyPart($etag));
|
||||
return;
|
||||
}
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_OK)
|
||||
->addHeader('ETag', '"' . $etag . '"')
|
||||
->send('');
|
||||
return;
|
||||
}
|
||||
|
||||
if ($request->getMethod() === Action::HTTP_REQUEST_METHOD_POST && $key !== '' && $this->query($request, 'uploadId') !== '') {
|
||||
$this->authorize($request, $project, $team, $user, ['files.write']);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
// No multipart lock here: clients complete only after every part
|
||||
// returned 200, and each part's state save commits inside its lock
|
||||
// before that response, so the snapshot below always holds all
|
||||
// acknowledged parts. Nesting the multipart lock around the object
|
||||
// lock inside completeMultipartUpload would also need two
|
||||
// connections from the fail-fast lock pool at once.
|
||||
$selected = $this->completedPartNumbers($this->requestBody($request));
|
||||
$upload = $this->multipartUploadForBucket($cache, $project, $this->query($request, 'uploadId'), $bucketId, $key);
|
||||
[$file, $event] = $this->completeMultipartUpload($locks, $dbForProject, $cache, $project, $deviceForFiles, $bucket, $upload, $selected);
|
||||
if ($event !== null) {
|
||||
$this->queueFileEvent($queueForEvents, $response, $bucket, $file, $event);
|
||||
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.' . $event, $file, 'file');
|
||||
}
|
||||
$this->sendXml($response, S3Xml::completeMultipart($bucketId, $key, $file->getAttribute('signature', '')));
|
||||
return;
|
||||
}
|
||||
|
||||
if ($key === '') {
|
||||
$this->authorize($request, $project, $team, $user, ['buckets.write']);
|
||||
$this->createBucket($dbForProject, $bucketId);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
$this->queueBucketEvent($queueForEvents, $response, $bucket, 'create');
|
||||
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'bucket.create', $bucket, 'bucket');
|
||||
$response->setStatusCode(Response::STATUS_CODE_OK)->send('');
|
||||
return;
|
||||
}
|
||||
|
||||
$this->authorize($request, $project, $team, $user, ['files.write']);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
if ($this->isFolderMarker($key)) {
|
||||
if ($request->getHeaderLine('x-amz-copy-source') !== '') {
|
||||
$this->sendXml($response, S3Xml::error('NotImplemented', 'Copying to an S3 folder marker is not supported.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
|
||||
return;
|
||||
}
|
||||
$this->objectPath($key . '.folder');
|
||||
$response->setStatusCode(Response::STATUS_CODE_OK)->send('');
|
||||
return;
|
||||
}
|
||||
|
||||
$copySource = $request->getHeaderLine('x-amz-copy-source');
|
||||
if ($copySource !== '') {
|
||||
// Copy reads the source bytes, so the key must also hold files.read.
|
||||
$this->authorize($request, $project, $team, $user, ['files.read']);
|
||||
$copySource = \rawurldecode(\ltrim($copySource, '/'));
|
||||
[$sourceBucketId, $sourceKey] = \array_pad(\explode('/', $copySource, 2), 2, '');
|
||||
$sourceBucket = $this->bucket($dbForProject, $sourceBucketId);
|
||||
$sourceFile = $this->getObject($dbForProject, $sourceBucket, $sourceKey);
|
||||
$this->validateCopySize($bucket, $sourceFile);
|
||||
$metadata = $sourceFile->getAttribute('metadata', []);
|
||||
if (\strtoupper($request->getHeaderLine('x-amz-metadata-directive')) === 'REPLACE') {
|
||||
$metadata = $this->requestObjectMetadata($request);
|
||||
}
|
||||
[$file, $event] = $this->putObjectLocked($locks, $dbForProject, $project, $deviceForFiles, $bucket, $key, (string) $deviceForFiles->read($sourceFile->getAttribute('path', '')), $sourceFile->getAttribute('mimeType', 'application/octet-stream'), $metadata);
|
||||
$this->queueFileEvent($queueForEvents, $response, $bucket, $file, $event);
|
||||
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.' . $event, $file, 'file');
|
||||
$this->sendXml($response, S3Xml::copyObject($file->getAttribute('signature', ''), $file->getUpdatedAt() ?: $file->getCreatedAt()));
|
||||
return;
|
||||
}
|
||||
|
||||
[$file, $event] = $this->putObjectLocked($locks, $dbForProject, $project, $deviceForFiles, $bucket, $key, $this->requestBody($request), $request->getHeaderLine('content-type', 'application/octet-stream'), $this->requestObjectMetadata($request));
|
||||
$this->queueFileEvent($queueForEvents, $response, $bucket, $file, $event);
|
||||
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.' . $event, $file, 'file');
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_OK)
|
||||
->addHeader('ETag', '"' . $file->getAttribute('signature', '') . '"');
|
||||
$sse = (string) ($file->getAttribute('metadata', [])['serverSideEncryption'] ?? '');
|
||||
if ($sse !== '') {
|
||||
$response->addHeader('x-amz-server-side-encryption', $sse);
|
||||
}
|
||||
$response->send('');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Http\S3;
|
||||
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Audit as AuditPublisher;
|
||||
use Appwrite\Event\Publisher\Delete as DeletePublisher;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Extend\Exception as AppwriteException;
|
||||
use Appwrite\Functions\EventProcessor;
|
||||
use Appwrite\Platform\Modules\S3\Responses\S3Xml;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Storage\Device;
|
||||
|
||||
class Delete extends Base
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 's3Delete';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct(Action::HTTP_REQUEST_METHOD_DELETE);
|
||||
}
|
||||
|
||||
protected function route(Request $request, Response $response, Database $dbForProject, Document $project, Document $team, User $user, Device $deviceForFiles, callable $locks, Cache $cache, Event $queueForEvents, DeletePublisher $publisherForDeletes, AuditPublisher $publisherForAudits, Event $queueForRealtime, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, EventProcessor $eventProcessor): void
|
||||
{
|
||||
[$bucketId, $key] = $this->parts($request);
|
||||
if ($bucketId === '') {
|
||||
throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Bucket is required.');
|
||||
}
|
||||
|
||||
if ($this->unsupportedSubresource($request)) {
|
||||
$this->authorize($request, $project, $team, $user, $key === '' ? ['buckets.write'] : ['files.write']);
|
||||
$this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
|
||||
return;
|
||||
}
|
||||
|
||||
if ($key === '') {
|
||||
$this->authorize($request, $project, $team, $user, ['buckets.write']);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
|
||||
$files = $dbForProject->getAuthorization()->skip(fn () => $dbForProject->find('bucket_' . $bucket->getSequence(), [Query::limit(1)]));
|
||||
if ($files !== []) {
|
||||
$this->sendXml($response, S3Xml::error('BucketNotEmpty', 'The bucket you tried to delete is not empty.', $request->getURI()), Response::STATUS_CODE_CONFLICT);
|
||||
return;
|
||||
}
|
||||
|
||||
$dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteCollection('bucket_' . $bucket->getSequence()));
|
||||
$dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('buckets', $bucketId));
|
||||
$this->queueBucketEvent($queueForEvents, $response, $bucket, 'delete');
|
||||
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'bucket.delete', $bucket, 'bucket');
|
||||
$response->noContent();
|
||||
return;
|
||||
}
|
||||
|
||||
$this->authorize($request, $project, $team, $user, ['files.write']);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
|
||||
if ($this->isFolderMarker($key)) {
|
||||
$response->noContent();
|
||||
return;
|
||||
}
|
||||
|
||||
if ($this->query($request, 'uploadId') !== '') {
|
||||
$uploadId = $this->query($request, 'uploadId');
|
||||
// Abort races in-flight part uploads for the same upload; take the
|
||||
// upload's critical section so a late part cannot resurrect state
|
||||
// that was just purged or upload into an aborted device staging.
|
||||
$locks($this->multipartLockKey($project, $uploadId), 600, function () use ($cache, $project, $deviceForFiles, $bucket, $uploadId, $bucketId, $key): void {
|
||||
$this->deleteMultipartUpload($cache, $project, $deviceForFiles, $bucket, $this->multipartUploadForBucket($cache, $project, $uploadId, $bucketId, $key));
|
||||
}, timeout: 120.0);
|
||||
$response->noContent();
|
||||
return;
|
||||
}
|
||||
|
||||
$file = $locks($this->objectLockKey($project, $bucket, $key), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $key): ?Document {
|
||||
$file = $this->findObject($dbForProject, $bucket, $key);
|
||||
if ($file === null) {
|
||||
return null;
|
||||
}
|
||||
if (!$deviceForFiles->delete($file->getAttribute('path', ''))) {
|
||||
throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Failed to delete object.');
|
||||
}
|
||||
$dbForProject->getAuthorization()->skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $file->getId()));
|
||||
|
||||
return $file;
|
||||
}, timeout: 120.0);
|
||||
if ($file !== null) {
|
||||
$this->enqueueFileCacheDelete($publisherForDeletes, $queueForEvents, $bucket, $file);
|
||||
$this->queueFileEvent($queueForEvents, $response, $bucket, $file, 'delete');
|
||||
$this->enqueueAudit($publisherForAudits, $request, $project, $user, 'file.delete', $file, 'file');
|
||||
}
|
||||
$response->noContent();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Http\S3;
|
||||
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Publisher\Audit as AuditPublisher;
|
||||
use Appwrite\Event\Publisher\Delete as DeletePublisher;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Functions\EventProcessor;
|
||||
use Appwrite\Platform\Modules\S3\Responses\S3Xml;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Cache\Cache;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Query;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Storage\Device;
|
||||
|
||||
class Get extends Base
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 's3Get';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct(Action::HTTP_REQUEST_METHOD_GET);
|
||||
}
|
||||
|
||||
protected function route(Request $request, Response $response, Database $dbForProject, Document $project, Document $team, User $user, Device $deviceForFiles, callable $locks, Cache $cache, Event $queueForEvents, DeletePublisher $publisherForDeletes, AuditPublisher $publisherForAudits, Event $queueForRealtime, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, EventProcessor $eventProcessor): void
|
||||
{
|
||||
$isHead = $request->getMethod() === Action::HTTP_REQUEST_METHOD_HEAD;
|
||||
[$bucketId, $key] = $this->parts($request);
|
||||
|
||||
if ($bucketId === '') {
|
||||
$this->authorize($request, $project, $team, $user, ['buckets.read']);
|
||||
if ($isHead) {
|
||||
$response->send('');
|
||||
return;
|
||||
}
|
||||
|
||||
$buckets = $dbForProject->getAuthorization()->skip(fn () => $dbForProject->find('buckets', [Query::limit(1000)]));
|
||||
$this->sendXml($response, S3Xml::listBuckets($buckets));
|
||||
return;
|
||||
}
|
||||
|
||||
if ($key === '') {
|
||||
$this->authorize($request, $project, $team, $user, ['files.read', 'buckets.read']);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
|
||||
if ($isHead) {
|
||||
$response->send('');
|
||||
return;
|
||||
}
|
||||
|
||||
if ($this->hasQuery($request, 'acl')) {
|
||||
$this->sendXml($response, S3Xml::accessControlPolicy());
|
||||
return;
|
||||
}
|
||||
if ($this->hasQuery($request, 'location')) {
|
||||
$this->sendXml($response, S3Xml::bucketLocation());
|
||||
return;
|
||||
}
|
||||
if ($this->unsupportedSubresource($request)) {
|
||||
$this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
|
||||
return;
|
||||
}
|
||||
if ($this->hasQuery($request, 'uploads')) {
|
||||
$this->sendXml($response, S3Xml::listMultipartUploads($bucketId, $this->getMultipartUploads($cache, $project, $bucket)));
|
||||
return;
|
||||
}
|
||||
|
||||
// iterate() treats the limit as its page size and follows cursors until every file is loaded.
|
||||
$files = $dbForProject->getAuthorization()->skip(fn () => \iterator_to_array($dbForProject->iterate('bucket_' . $bucket->getSequence(), [
|
||||
Query::orderAsc('$sequence'),
|
||||
Query::limit(1000),
|
||||
]), false));
|
||||
$this->assertUniqueObjectKeys($files);
|
||||
if ($this->query($request, 'list-type') === '2') {
|
||||
$this->sendXml($response, S3Xml::listObjectsV2(
|
||||
$bucketId,
|
||||
$files,
|
||||
$this->query($request, 'prefix'),
|
||||
$this->query($request, 'delimiter'),
|
||||
(int) $this->query($request, 'max-keys', '1000'),
|
||||
$this->query($request, 'continuation-token'),
|
||||
$this->query($request, 'start-after')
|
||||
));
|
||||
return;
|
||||
}
|
||||
|
||||
$this->sendXml($response, S3Xml::listObjects(
|
||||
$bucketId,
|
||||
$files,
|
||||
$this->query($request, 'prefix'),
|
||||
$this->query($request, 'delimiter'),
|
||||
(int) $this->query($request, 'max-keys', '1000'),
|
||||
$this->query($request, 'marker')
|
||||
));
|
||||
return;
|
||||
}
|
||||
|
||||
$this->authorize($request, $project, $team, $user, ['files.read']);
|
||||
$bucket = $this->bucket($dbForProject, $bucketId);
|
||||
|
||||
if ($this->query($request, 'uploadId') !== '') {
|
||||
$parts = $this->getMultipartParts($this->multipartUploadForBucket($cache, $project, $this->query($request, 'uploadId'), $bucketId, $key));
|
||||
$this->sendXml($response, S3Xml::listParts($bucketId, $key, $this->query($request, 'uploadId'), $parts));
|
||||
return;
|
||||
}
|
||||
|
||||
$isAcl = $this->hasQuery($request, 'acl');
|
||||
$isUnsupported = $this->unsupportedSubresource($request);
|
||||
$ifNoneMatch = $request->getHeaderLine('if-none-match');
|
||||
$ifMatch = $request->getHeaderLine('if-match');
|
||||
// Ordinary PUT overwrites its Device path in place, so metadata lookup and
|
||||
// body materialization must not overlap that key's write critical section.
|
||||
[$file, $body, $changedRepeatedly] = $locks($this->objectLockKey($project, $bucket, $key), 600, function () use ($dbForProject, $deviceForFiles, $bucket, $key, $isHead, $isAcl, $isUnsupported, $ifNoneMatch, $ifMatch, $request): array {
|
||||
$file = $this->getObject($dbForProject, $bucket, $key);
|
||||
$body = null;
|
||||
$retried = false;
|
||||
$changedRepeatedly = false;
|
||||
while (true) {
|
||||
$path = $file->getAttribute('path', '');
|
||||
$size = (int) $file->getAttribute('sizeOriginal', 0);
|
||||
$range = $this->range($request, $size);
|
||||
$etag = '"' . $file->getAttribute('signature', '') . '"';
|
||||
|
||||
if (
|
||||
$isHead
|
||||
|| $isAcl
|
||||
|| $isUnsupported
|
||||
|| $ifNoneMatch === $etag
|
||||
|| ($ifMatch !== '' && $ifMatch !== $etag)
|
||||
|| $range === false
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
$body = (string) $deviceForFiles->read($path);
|
||||
break;
|
||||
} catch (\Throwable $error) {
|
||||
$latest = $this->getObject($dbForProject, $bucket, $key);
|
||||
if ($latest->getAttribute('path', '') === $path) {
|
||||
throw $error;
|
||||
}
|
||||
// One moved-path retry closes races with writes outside the S3 gateway.
|
||||
if ($retried) {
|
||||
$changedRepeatedly = true;
|
||||
break;
|
||||
}
|
||||
$retried = true;
|
||||
$file = $latest;
|
||||
}
|
||||
}
|
||||
|
||||
return [$file, $body, $changedRepeatedly];
|
||||
}, timeout: 120.0);
|
||||
|
||||
if ($changedRepeatedly) {
|
||||
$this->sendXml($response, S3Xml::error('SlowDown', 'The object changed repeatedly while it was being read. Please retry.', $request->getURI()), Response::STATUS_CODE_SERVICE_UNAVAILABLE);
|
||||
return;
|
||||
}
|
||||
|
||||
if ($isAcl) {
|
||||
$this->sendXml($response, S3Xml::accessControlPolicy());
|
||||
return;
|
||||
}
|
||||
if ($isUnsupported) {
|
||||
$this->sendXml($response, S3Xml::error('NotImplemented', 'This S3 feature is not supported by Appwrite Storage.', $request->getURI()), Response::STATUS_CODE_NOT_IMPLEMENTED);
|
||||
return;
|
||||
}
|
||||
|
||||
$size = (int) $file->getAttribute('sizeOriginal', 0);
|
||||
$range = $this->range($request, $size);
|
||||
$etag = '"' . $file->getAttribute('signature', '') . '"';
|
||||
|
||||
if ($ifNoneMatch === $etag) {
|
||||
$response->setStatusCode(Response::STATUS_CODE_NOT_MODIFIED)->send('');
|
||||
return;
|
||||
}
|
||||
if ($ifMatch !== '' && $ifMatch !== $etag) {
|
||||
$this->sendXml($response, S3Xml::error('PreconditionFailed', 'At least one of the pre-conditions you specified did not hold.', $request->getURI()), Response::STATUS_CODE_PRECONDITION_FAILED);
|
||||
return;
|
||||
}
|
||||
|
||||
if ($range === false) {
|
||||
$response->addHeader('Content-Range', 'bytes */' . $size);
|
||||
$this->sendXml($response, S3Xml::error('InvalidRange', 'The requested range is not satisfiable.', $request->getURI()), Response::STATUS_CODE_REQUESTED_RANGE_NOT_SATISFIABLE);
|
||||
return;
|
||||
}
|
||||
|
||||
$response
|
||||
->setContentType($file->getAttribute('mimeType', 'application/octet-stream'))
|
||||
->addHeader('ETag', $etag)
|
||||
->addHeader('Accept-Ranges', 'bytes');
|
||||
|
||||
$lastModified = \strtotime($file->getUpdatedAt() ?: $file->getCreatedAt());
|
||||
if ($lastModified !== false) {
|
||||
$response->addHeader('Last-Modified', \gmdate('D, d M Y H:i:s', $lastModified) . ' GMT');
|
||||
}
|
||||
|
||||
$metadata = $file->getAttribute('metadata', []);
|
||||
foreach (($metadata['userMetadata'] ?? []) as $name => $value) {
|
||||
$response->addHeader('x-amz-meta-' . $name, (string) $value);
|
||||
}
|
||||
$sse = (string) ($metadata['serverSideEncryption'] ?? '');
|
||||
if ($sse !== '') {
|
||||
$response->addHeader('x-amz-server-side-encryption', $sse);
|
||||
}
|
||||
|
||||
foreach ([
|
||||
'cacheControl' => 'Cache-Control',
|
||||
'contentDisposition' => 'Content-Disposition',
|
||||
'contentEncoding' => 'Content-Encoding',
|
||||
'contentLanguage' => 'Content-Language',
|
||||
'expires' => 'Expires',
|
||||
] as $attribute => $header) {
|
||||
$value = (string) ($metadata[$attribute] ?? '');
|
||||
if ($value !== '') {
|
||||
$response->addHeader($header, $value);
|
||||
}
|
||||
}
|
||||
|
||||
foreach ([
|
||||
'response-cache-control' => 'Cache-Control',
|
||||
'response-content-disposition' => 'Content-Disposition',
|
||||
'response-content-encoding' => 'Content-Encoding',
|
||||
'response-content-language' => 'Content-Language',
|
||||
'response-expires' => 'Expires',
|
||||
] as $parameter => $header) {
|
||||
$value = $this->query($request, $parameter);
|
||||
if ($value !== '') {
|
||||
$response->setHeader($header, $value);
|
||||
}
|
||||
}
|
||||
|
||||
$responseContentType = $this->query($request, 'response-content-type');
|
||||
if ($responseContentType !== '') {
|
||||
$response->setContentType($responseContentType);
|
||||
}
|
||||
|
||||
if ($range !== null) {
|
||||
[$start, $end] = $range;
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT)
|
||||
->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size)
|
||||
->addHeader('Content-Length', (string) ($end - $start + 1));
|
||||
} else {
|
||||
$response->addHeader('Content-Length', (string) $size);
|
||||
}
|
||||
|
||||
if ($isHead) {
|
||||
$response->send('');
|
||||
return;
|
||||
}
|
||||
|
||||
if ($range !== null) {
|
||||
[$start, $end] = $range;
|
||||
$body = \substr((string) $body, $start, $end - $start + 1);
|
||||
}
|
||||
|
||||
$response->send((string) $body);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0: int, 1: int}|false|null
|
||||
*/
|
||||
private function range(Request $request, int $size): array|false|null
|
||||
{
|
||||
$header = \trim($request->getHeaderLine('range'));
|
||||
if ($header === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!\preg_match('/^bytes=(\d*)-(\d*)$/', $header, $matches)) {
|
||||
return false;
|
||||
}
|
||||
if ($size <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$start = $matches[1];
|
||||
$end = $matches[2];
|
||||
if ($start === '' && $end === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($start === '') {
|
||||
$suffix = (int) $end;
|
||||
if ($suffix <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return [\max(0, $size - $suffix), \max(0, $size - 1)];
|
||||
}
|
||||
|
||||
$start = (int) $start;
|
||||
$end = $end === '' ? $size - 1 : (int) $end;
|
||||
|
||||
if ($start >= $size || $start > $end) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return [$start, \min($end, $size - 1)];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Http\Shutdown;
|
||||
|
||||
use Appwrite\Event\Event;
|
||||
use Appwrite\Event\Message\Func as FunctionMessage;
|
||||
use Appwrite\Event\Publisher\Func as FunctionPublisher;
|
||||
use Appwrite\Event\Realtime;
|
||||
use Appwrite\Functions\EventProcessor;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Database\Database;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Http\Route;
|
||||
use Utopia\Platform\Action;
|
||||
|
||||
/**
|
||||
* Fans queued S3 events out to realtime, event-triggered functions and
|
||||
* webhooks. Mirrors the `api` group's events shutdown hook in
|
||||
* app/controllers/shared/api.php, which does not run for the `s3` group.
|
||||
*/
|
||||
class Events extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'shutdownEvents';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->setType(Action::TYPE_SHUTDOWN)
|
||||
->groups(['s3'])
|
||||
->desc('S3 gateway event fan-out')
|
||||
->inject('route')
|
||||
->inject('response')
|
||||
->inject('project')
|
||||
->inject('queueForEvents')
|
||||
->inject('publisherForFunctions')
|
||||
->inject('queueForWebhooks')
|
||||
->inject('queueForRealtime')
|
||||
->inject('dbForProject')
|
||||
->inject('eventProcessor')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(Route $route, Response $response, Document $project, Event $queueForEvents, FunctionPublisher $publisherForFunctions, Event $queueForWebhooks, Realtime $queueForRealtime, Database $dbForProject, EventProcessor $eventProcessor): void
|
||||
{
|
||||
if (empty($queueForEvents->getEvent())) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (empty($queueForEvents->getPayload())) {
|
||||
$queueForEvents->setPayload($response->getPayload());
|
||||
}
|
||||
|
||||
// Get project and function/webhook events (cached)
|
||||
$functionsEvents = $eventProcessor->getFunctionsEvents($project, $dbForProject);
|
||||
$webhooksEvents = $eventProcessor->getWebhooksEvents($project);
|
||||
|
||||
// Generate events for this operation
|
||||
$generatedEvents = Event::generateEvents(
|
||||
$queueForEvents->getEvent(),
|
||||
$queueForEvents->getParams()
|
||||
);
|
||||
|
||||
$allowedOnConsole = !empty(\array_intersect($route->getGroups(), Realtime::CONSOLE_ALLOWLIST));
|
||||
if ($project->getId() !== 'console' || $allowedOnConsole) {
|
||||
$queueForRealtime
|
||||
->from($queueForEvents)
|
||||
->trigger();
|
||||
}
|
||||
|
||||
// Only trigger functions if there are matching function events
|
||||
if (!empty($functionsEvents)) {
|
||||
foreach ($generatedEvents as $event) {
|
||||
if (isset($functionsEvents[$event])) {
|
||||
$publisherForFunctions->enqueue(FunctionMessage::fromEvent(
|
||||
event: $queueForEvents->getEvent(),
|
||||
params: $queueForEvents->getParams(),
|
||||
project: $queueForEvents->getProject(),
|
||||
user: $queueForEvents->getUser(),
|
||||
userId: $queueForEvents->getUserId(),
|
||||
payload: $queueForEvents->getPayload(),
|
||||
platform: $queueForEvents->getPlatform(),
|
||||
));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only trigger webhooks if there are matching webhook events
|
||||
if (!empty($webhooksEvents)) {
|
||||
foreach ($generatedEvents as $event) {
|
||||
if (isset($webhooksEvents[$event])) {
|
||||
$queueForWebhooks
|
||||
->from($queueForEvents)
|
||||
->trigger();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Http\Shutdown;
|
||||
|
||||
use Appwrite\Bus\Events\RequestCompleted;
|
||||
use Appwrite\Event\Message\Usage as UsageMessage;
|
||||
use Appwrite\Event\Publisher\Usage as UsagePublisher;
|
||||
use Appwrite\Usage\Context as UsageContext;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use Appwrite\Utopia\Request;
|
||||
use Appwrite\Utopia\Response;
|
||||
use Utopia\Bus\Bus;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Platform\Action;
|
||||
|
||||
/**
|
||||
* Meters S3 gateway traffic. Mirrors the `api` group's usage shutdown hook in
|
||||
* app/controllers/shared/api.php, which does not run for the `s3`
|
||||
* group.
|
||||
*/
|
||||
class Usage extends Action
|
||||
{
|
||||
public static function getName(): string
|
||||
{
|
||||
return 'shutdownUsage';
|
||||
}
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this
|
||||
->setType(Action::TYPE_SHUTDOWN)
|
||||
->groups(['s3'])
|
||||
->desc('S3 gateway usage metering')
|
||||
->inject('request')
|
||||
->inject('response')
|
||||
->inject('project')
|
||||
->inject('user')
|
||||
->inject('usage')
|
||||
->inject('publisherForUsage')
|
||||
->inject('authorization')
|
||||
->inject('bus')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
public function action(Request $request, Response $response, Document $project, User $user, UsageContext $usage, UsagePublisher $publisherForUsage, Authorization $authorization, Bus $bus): void
|
||||
{
|
||||
if ($project->isEmpty() || $project->getId() === 'console') {
|
||||
return;
|
||||
}
|
||||
|
||||
$usage->setStatus($response->getStatusCode());
|
||||
if ($usage->getResourcePath() === '') {
|
||||
$usage->fillMissingResource('project', $project->getId(), (string) $project->getSequence());
|
||||
}
|
||||
|
||||
// The usage bus listener adds METRIC_NETWORK_REQUESTS/INBOUND/OUTBOUND,
|
||||
// exactly as on the `api` group. The SigV4 key never becomes the
|
||||
// `apiKey` resource, so per-key disabled-metrics filtering cannot
|
||||
// apply here.
|
||||
if (!$user->isPrivileged($authorization->getRoles())) {
|
||||
$bus->dispatch(new RequestCompleted(
|
||||
project: $project->getArrayCopy(),
|
||||
request: $request,
|
||||
response: $response,
|
||||
));
|
||||
}
|
||||
|
||||
if (!$usage->isEmpty()) {
|
||||
$publisherForUsage->enqueue(new UsageMessage(
|
||||
project: $project,
|
||||
metrics: $usage->getMetrics(),
|
||||
reduce: $usage->getReduce(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3;
|
||||
|
||||
use Appwrite\Platform\Modules\S3\Services\Http;
|
||||
use Utopia\Platform;
|
||||
|
||||
class Module extends Platform\Module
|
||||
{
|
||||
public function __construct()
|
||||
{
|
||||
$this->addService('http', new Http());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Requests;
|
||||
|
||||
use Appwrite\Extend\Exception as AppwriteException;
|
||||
|
||||
/**
|
||||
* Decoder for aws-chunked request bodies (AWS SigV4 streaming payloads).
|
||||
*
|
||||
* Modern AWS SDKs upload objects as `Content-Encoding: aws-chunked` with
|
||||
* `x-amz-content-sha256: STREAMING-*`: the body arrives framed as
|
||||
* `<hex-size>[;chunk-signature=…]\r\n<bytes>\r\n` chunks, terminated by a
|
||||
* zero-size chunk and optional trailing headers. Storing the raw payload
|
||||
* would persist the framing bytes as object content.
|
||||
*
|
||||
* Per-chunk signatures are stripped without verification — the request is
|
||||
* already authenticated by the seed signature over the canonical request,
|
||||
* which pins the STREAMING-* payload mode. Integrity is checked through the
|
||||
* declared decoded length and, when present, the CRC32 trailer checksum.
|
||||
*/
|
||||
class AwsChunked
|
||||
{
|
||||
public static function applies(string $contentSha256, string $contentEncoding): bool
|
||||
{
|
||||
if (\str_starts_with(\strtoupper(\trim($contentSha256)), 'STREAMING-')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
foreach (\explode(',', $contentEncoding) as $encoding) {
|
||||
if (\strtolower(\trim($encoding)) === 'aws-chunked') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static function decode(string $payload, ?int $declaredLength = null): string
|
||||
{
|
||||
$decoded = '';
|
||||
$offset = 0;
|
||||
$length = \strlen($payload);
|
||||
|
||||
while (true) {
|
||||
if ($offset >= $length) {
|
||||
throw self::malformed('Missing final zero-size chunk.');
|
||||
}
|
||||
|
||||
$lineEnd = \strpos($payload, "\r\n", $offset);
|
||||
if ($lineEnd === false) {
|
||||
throw self::malformed('Chunk header is not CRLF-terminated.');
|
||||
}
|
||||
|
||||
$sizeHex = \explode(';', \substr($payload, $offset, $lineEnd - $offset), 2)[0];
|
||||
if ($sizeHex === '' || \strlen($sizeHex) > 8 || \preg_match('/^[0-9a-fA-F]+$/', $sizeHex) !== 1) {
|
||||
throw self::malformed('Chunk size is not hexadecimal.');
|
||||
}
|
||||
|
||||
$size = (int) \hexdec($sizeHex);
|
||||
$offset = $lineEnd + 2;
|
||||
|
||||
if ($size === 0) {
|
||||
self::verifyTrailers(self::trailers($payload, $offset), $decoded);
|
||||
break;
|
||||
}
|
||||
|
||||
if ($offset + $size > $length) {
|
||||
throw self::malformed('Chunk data is truncated.');
|
||||
}
|
||||
|
||||
$decoded .= \substr($payload, $offset, $size);
|
||||
$offset += $size;
|
||||
|
||||
if (\substr($payload, $offset, 2) !== "\r\n") {
|
||||
throw self::malformed('Chunk data is not CRLF-terminated.');
|
||||
}
|
||||
$offset += 2;
|
||||
}
|
||||
|
||||
if ($declaredLength !== null && \strlen($decoded) !== $declaredLength) {
|
||||
throw self::malformed('Decoded length does not match x-amz-decoded-content-length.');
|
||||
}
|
||||
|
||||
return $decoded;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, string>
|
||||
*/
|
||||
private static function trailers(string $payload, int $offset): array
|
||||
{
|
||||
$trailers = [];
|
||||
$length = \strlen($payload);
|
||||
|
||||
while ($offset < $length) {
|
||||
$lineEnd = \strpos($payload, "\r\n", $offset);
|
||||
$line = $lineEnd === false ? \substr($payload, $offset) : \substr($payload, $offset, $lineEnd - $offset);
|
||||
$offset = $lineEnd === false ? $length : $lineEnd + 2;
|
||||
|
||||
if (\trim($line) === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
[$name, $value] = \array_pad(\explode(':', $line, 2), 2, '');
|
||||
$trailers[\strtolower(\trim($name))] = \trim($value);
|
||||
}
|
||||
|
||||
return $trailers;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, string> $trailers
|
||||
*/
|
||||
private static function verifyTrailers(array $trailers, string $decoded): void
|
||||
{
|
||||
// CRC32 is the SDK default checksum algorithm; other algorithms
|
||||
// (crc32c, sha1, sha256, crc64nvme) are accepted but not verified.
|
||||
$crc32 = $trailers['x-amz-checksum-crc32'] ?? '';
|
||||
if ($crc32 !== '' && \base64_encode(\hash('crc32b', $decoded, true)) !== $crc32) {
|
||||
throw self::malformed('CRC32 trailer checksum does not match the decoded payload.');
|
||||
}
|
||||
}
|
||||
|
||||
private static function malformed(string $reason): AppwriteException
|
||||
{
|
||||
return new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, 'Malformed aws-chunked payload. ' . $reason);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Responses;
|
||||
|
||||
use Utopia\Database\Document;
|
||||
|
||||
class S3Xml
|
||||
{
|
||||
public static function error(string $code, string $message, string $resource = '', string $requestId = ''): string
|
||||
{
|
||||
return self::document('Error', [
|
||||
'Code' => $code,
|
||||
'Message' => $message,
|
||||
'Resource' => $resource,
|
||||
'RequestId' => $requestId ?: \bin2hex(\random_bytes(8)),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Document> $buckets
|
||||
*/
|
||||
public static function listBuckets(array $buckets): string
|
||||
{
|
||||
$items = '';
|
||||
foreach ($buckets as $bucket) {
|
||||
$items .= self::element('Bucket', [
|
||||
'Name' => $bucket->getId(),
|
||||
'CreationDate' => self::date($bucket->getCreatedAt()),
|
||||
]);
|
||||
}
|
||||
|
||||
return self::rawDocument('ListAllMyBucketsResult', '<Buckets>' . $items . '</Buckets>');
|
||||
}
|
||||
|
||||
public static function accessControlPolicy(): string
|
||||
{
|
||||
return self::rawDocument('AccessControlPolicy', self::element('Owner', [
|
||||
'ID' => 'appwrite',
|
||||
'DisplayName' => 'appwrite',
|
||||
]) . '<AccessControlList>' . self::element('Grant', [
|
||||
'Grantee' => [
|
||||
'ID' => 'appwrite',
|
||||
'DisplayName' => 'appwrite',
|
||||
],
|
||||
'Permission' => 'FULL_CONTROL',
|
||||
]) . '</AccessControlList>');
|
||||
}
|
||||
|
||||
public static function bucketLocation(string $region = 'us-east-1'): string
|
||||
{
|
||||
return self::rawDocument('LocationConstraint', $region === 'us-east-1' ? '' : self::escape($region));
|
||||
}
|
||||
|
||||
public static function copyObject(string $etag, ?string $lastModified = null): string
|
||||
{
|
||||
return self::document('CopyObjectResult', [
|
||||
'LastModified' => self::date($lastModified),
|
||||
'ETag' => '"' . $etag . '"',
|
||||
]);
|
||||
}
|
||||
|
||||
public static function copyPart(string $etag, ?string $lastModified = null): string
|
||||
{
|
||||
return self::document('CopyPartResult', [
|
||||
'LastModified' => self::date($lastModified),
|
||||
'ETag' => '"' . $etag . '"',
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $keys
|
||||
*/
|
||||
public static function deleteObjects(array $keys, bool $quiet = false): string
|
||||
{
|
||||
$items = '';
|
||||
if (!$quiet) {
|
||||
foreach ($keys as $key) {
|
||||
$items .= self::element('Deleted', ['Key' => $key]);
|
||||
}
|
||||
}
|
||||
|
||||
return self::rawDocument('DeleteResult', $items);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, array{key: string, uploadId: string, initiated: string}> $uploads
|
||||
*/
|
||||
public static function listMultipartUploads(string $bucketId, array $uploads): string
|
||||
{
|
||||
$items = '';
|
||||
foreach ($uploads as $upload) {
|
||||
$items .= self::element('Upload', [
|
||||
'Key' => $upload['key'],
|
||||
'UploadId' => $upload['uploadId'],
|
||||
'Initiated' => self::date($upload['initiated']),
|
||||
]);
|
||||
}
|
||||
|
||||
return self::rawDocument('ListMultipartUploadsResult', self::element('Bucket', $bucketId) . self::element('IsTruncated', 'false') . $items);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Document> $files
|
||||
*/
|
||||
public static function listObjects(string $bucketId, array $files, string $prefix = '', string $delimiter = '', int $maxKeys = 1000, string $marker = ''): string
|
||||
{
|
||||
$maxKeys = \max(0, \min(1000, $maxKeys));
|
||||
$entries = [];
|
||||
$commonPrefixes = [];
|
||||
foreach ($files as $file) {
|
||||
$key = self::fileKey($file);
|
||||
if (($prefix !== '' && !\str_starts_with($key, $prefix)) || ($marker !== '' && \strcmp($key, $marker) <= 0)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($delimiter !== '') {
|
||||
$rest = \substr($key, \strlen($prefix));
|
||||
$position = \strpos($rest, $delimiter);
|
||||
if ($position !== false) {
|
||||
$commonPrefixes[$prefix . \substr($rest, 0, $position + \strlen($delimiter))] = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
$entries[$key] = ['type' => 'file', 'file' => $file];
|
||||
}
|
||||
|
||||
foreach (\array_keys($commonPrefixes) as $commonPrefix) {
|
||||
if ($marker !== '' && \strcmp($commonPrefix, $marker) <= 0) {
|
||||
continue;
|
||||
}
|
||||
$entries[$commonPrefix] = ['type' => 'prefix', 'prefix' => $commonPrefix];
|
||||
}
|
||||
|
||||
\ksort($entries, SORT_STRING);
|
||||
$selected = \array_slice($entries, 0, $maxKeys, true);
|
||||
$isTruncated = \count($entries) > $maxKeys;
|
||||
$lastKey = $selected === [] ? '' : (string) \array_key_last($selected);
|
||||
$items = '';
|
||||
foreach ($selected as $key => $entry) {
|
||||
if ($entry['type'] === 'prefix') {
|
||||
$items .= self::element('CommonPrefixes', ['Prefix' => (string) $entry['prefix']]);
|
||||
continue;
|
||||
}
|
||||
|
||||
$file = $entry['file'];
|
||||
$items .= self::element('Contents', [
|
||||
'Key' => $key,
|
||||
'LastModified' => self::date($file->getUpdatedAt() ?: $file->getCreatedAt()),
|
||||
'ETag' => '"' . $file->getAttribute('signature', '') . '"',
|
||||
'Size' => (string) $file->getAttribute('sizeOriginal', 0),
|
||||
'StorageClass' => 'STANDARD',
|
||||
]);
|
||||
}
|
||||
|
||||
$body = self::elements([
|
||||
'Name' => $bucketId,
|
||||
'Prefix' => $prefix,
|
||||
'Marker' => $marker,
|
||||
'MaxKeys' => (string) $maxKeys,
|
||||
'IsTruncated' => $isTruncated ? 'true' : 'false',
|
||||
]);
|
||||
if ($delimiter !== '') {
|
||||
$body .= self::element('Delimiter', $delimiter);
|
||||
}
|
||||
if ($isTruncated && $lastKey !== '') {
|
||||
$body .= self::element('NextMarker', $lastKey);
|
||||
}
|
||||
|
||||
return self::rawDocument('ListBucketResult', $body . $items);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<Document> $files
|
||||
*/
|
||||
public static function listObjectsV2(
|
||||
string $bucketId,
|
||||
array $files,
|
||||
string $prefix = '',
|
||||
string $delimiter = '',
|
||||
int $maxKeys = 1000,
|
||||
string $continuationToken = '',
|
||||
string $startAfter = ''
|
||||
): string {
|
||||
$maxKeys = \max(0, \min(1000, $maxKeys));
|
||||
$after = $continuationToken !== '' ? self::decodeContinuationToken($continuationToken) : $startAfter;
|
||||
$entries = [];
|
||||
$commonPrefixes = [];
|
||||
|
||||
foreach ($files as $file) {
|
||||
$key = self::fileKey($file);
|
||||
if ($prefix !== '' && !\str_starts_with($key, $prefix)) {
|
||||
continue;
|
||||
}
|
||||
if ($after !== '' && \strcmp($key, $after) <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($delimiter !== '') {
|
||||
$rest = \substr($key, \strlen($prefix));
|
||||
$position = \strpos($rest, $delimiter);
|
||||
if ($position !== false) {
|
||||
$commonPrefixes[$prefix . \substr($rest, 0, $position + \strlen($delimiter))] = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
$entries[$key] = ['type' => 'file', 'file' => $file];
|
||||
}
|
||||
|
||||
foreach (\array_keys($commonPrefixes) as $commonPrefix) {
|
||||
if ($after !== '' && \strcmp($commonPrefix, $after) <= 0) {
|
||||
continue;
|
||||
}
|
||||
$entries[$commonPrefix] = ['type' => 'prefix', 'prefix' => $commonPrefix];
|
||||
}
|
||||
|
||||
// PHP casts numeric keys to int; without SORT_STRING they sort numerically,
|
||||
// which breaks the strcmp-based continuation filter and skips objects.
|
||||
\ksort($entries, SORT_STRING);
|
||||
|
||||
$selected = \array_slice($entries, 0, $maxKeys, true);
|
||||
$isTruncated = \count($entries) > $maxKeys;
|
||||
$lastKey = $selected === [] ? '' : (string) \array_key_last($selected);
|
||||
|
||||
$items = '';
|
||||
foreach ($selected as $entry) {
|
||||
if ($entry['type'] === 'prefix') {
|
||||
$items .= self::element('CommonPrefixes', [
|
||||
'Prefix' => (string) $entry['prefix'],
|
||||
]);
|
||||
continue;
|
||||
}
|
||||
|
||||
$file = $entry['file'];
|
||||
$items .= self::element('Contents', [
|
||||
'Key' => self::fileKey($file),
|
||||
'LastModified' => self::date($file->getUpdatedAt() ?: $file->getCreatedAt()),
|
||||
'ETag' => '"' . $file->getAttribute('signature', '') . '"',
|
||||
'Size' => (string) $file->getAttribute('sizeOriginal', 0),
|
||||
'StorageClass' => 'STANDARD',
|
||||
]);
|
||||
}
|
||||
|
||||
$body = self::elements([
|
||||
'Name' => $bucketId,
|
||||
'Prefix' => $prefix,
|
||||
'KeyCount' => (string) \count($selected),
|
||||
'MaxKeys' => (string) $maxKeys,
|
||||
'IsTruncated' => $isTruncated ? 'true' : 'false',
|
||||
]);
|
||||
|
||||
if ($delimiter !== '') {
|
||||
$body .= self::element('Delimiter', $delimiter);
|
||||
}
|
||||
if ($continuationToken !== '') {
|
||||
$body .= self::element('ContinuationToken', $continuationToken);
|
||||
}
|
||||
if ($startAfter !== '') {
|
||||
$body .= self::element('StartAfter', $startAfter);
|
||||
}
|
||||
if ($isTruncated && $lastKey !== '') {
|
||||
$body .= self::element('NextContinuationToken', self::encodeContinuationToken($lastKey));
|
||||
}
|
||||
|
||||
return self::rawDocument('ListBucketResult', $body . $items);
|
||||
}
|
||||
|
||||
public static function initiateMultipart(string $bucketId, string $key, string $uploadId): string
|
||||
{
|
||||
return self::document('InitiateMultipartUploadResult', [
|
||||
'Bucket' => $bucketId,
|
||||
'Key' => $key,
|
||||
'UploadId' => $uploadId,
|
||||
]);
|
||||
}
|
||||
|
||||
public static function completeMultipart(string $bucketId, string $key, string $etag): string
|
||||
{
|
||||
return self::document('CompleteMultipartUploadResult', [
|
||||
'Location' => '/' . $bucketId . '/' . $key,
|
||||
'Bucket' => $bucketId,
|
||||
'Key' => $key,
|
||||
'ETag' => '"' . $etag . '"',
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, array{partNumber: int, etag: string, size: int}> $parts
|
||||
*/
|
||||
public static function listParts(string $bucketId, string $key, string $uploadId, array $parts): string
|
||||
{
|
||||
$items = '';
|
||||
foreach ($parts as $part) {
|
||||
$items .= self::element('Part', [
|
||||
'PartNumber' => (string) $part['partNumber'],
|
||||
'ETag' => '"' . $part['etag'] . '"',
|
||||
'Size' => (string) $part['size'],
|
||||
]);
|
||||
}
|
||||
|
||||
return self::rawDocument('ListPartsResult', self::elements([
|
||||
'Bucket' => $bucketId,
|
||||
'Key' => $key,
|
||||
'UploadId' => $uploadId,
|
||||
]) . $items);
|
||||
}
|
||||
|
||||
private static function document(string $root, array $values): string
|
||||
{
|
||||
return self::rawDocument($root, self::elements($values));
|
||||
}
|
||||
|
||||
private static function rawDocument(string $root, string $body): string
|
||||
{
|
||||
return '<?xml version="1.0" encoding="UTF-8"?>' . '<' . $root . '>' . $body . '</' . $root . '>';
|
||||
}
|
||||
|
||||
private static function elements(array $values): string
|
||||
{
|
||||
$xml = '';
|
||||
foreach ($values as $key => $value) {
|
||||
$xml .= self::element((string) $key, \is_array($value) ? $value : (string) $value);
|
||||
}
|
||||
return $xml;
|
||||
}
|
||||
|
||||
private static function element(string $name, string|array $value): string
|
||||
{
|
||||
$content = \is_array($value) ? self::elements($value) : self::escape($value);
|
||||
return '<' . $name . '>' . $content . '</' . $name . '>';
|
||||
}
|
||||
|
||||
private static function fileKey(Document $file): string
|
||||
{
|
||||
return $file->getAttribute('folder', '') . $file->getAttribute('name', $file->getId());
|
||||
}
|
||||
|
||||
private static function encodeContinuationToken(string $key): string
|
||||
{
|
||||
return \rtrim(\strtr(\base64_encode($key), '+/', '-_'), '=');
|
||||
}
|
||||
|
||||
private static function decodeContinuationToken(string $token): string
|
||||
{
|
||||
$decoded = \base64_decode(\strtr($token, '-_', '+/'), true);
|
||||
return \is_string($decoded) ? $decoded : '';
|
||||
}
|
||||
|
||||
private static function escape(string $value): string
|
||||
{
|
||||
return \htmlspecialchars($value, ENT_XML1 | ENT_COMPAT, 'UTF-8');
|
||||
}
|
||||
|
||||
private static function date(?string $date): string
|
||||
{
|
||||
$time = \strtotime($date ?? '');
|
||||
return \gmdate('Y-m-d\TH:i:s.000\Z', $time === false ? \time() : $time);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<?php
|
||||
|
||||
namespace Appwrite\Platform\Modules\S3\Services;
|
||||
|
||||
use Appwrite\Platform\Modules\S3\Http\Init;
|
||||
use Appwrite\Platform\Modules\S3\Http\S3\Create;
|
||||
use Appwrite\Platform\Modules\S3\Http\S3\Delete;
|
||||
use Appwrite\Platform\Modules\S3\Http\S3\Get;
|
||||
use Appwrite\Platform\Modules\S3\Http\Shutdown\Events as ShutdownEvents;
|
||||
use Appwrite\Platform\Modules\S3\Http\Shutdown\Usage as ShutdownUsage;
|
||||
use Utopia\Platform\Service;
|
||||
|
||||
class Http extends Service
|
||||
{
|
||||
public function __construct()
|
||||
{
|
||||
$this->type = Service::TYPE_HTTP;
|
||||
|
||||
$this->addAction(Init::getName(), new Init());
|
||||
$this->addAction(Get::getName(), new Get());
|
||||
$this->addAction(Create::getName(), new Create());
|
||||
$this->addAction(Delete::getName(), new Delete());
|
||||
$this->addAction(ShutdownEvents::getName(), new ShutdownEvents());
|
||||
$this->addAction(ShutdownUsage::getName(), new ShutdownUsage());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Platform\Modules\S3\Auth;
|
||||
|
||||
use Appwrite\Extend\Exception as AppwriteException;
|
||||
use Appwrite\Platform\Modules\S3\Auth\SignatureV4;
|
||||
use Appwrite\Utopia\Database\Documents\User;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Swoole\Http\Request as SwooleRequest;
|
||||
use Utopia\Database\Document;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
|
||||
final class SignatureV4Test extends TestCase
|
||||
{
|
||||
public function testVerifiesValidSignatureForExistingProjectApiKey(): void
|
||||
{
|
||||
$secret = 'test-api-key-secret';
|
||||
$request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret);
|
||||
|
||||
$key = (new SignatureV4())->verify(
|
||||
$request,
|
||||
$this->project($secret, ['files.read']),
|
||||
new Document(),
|
||||
new User(),
|
||||
['files.read']
|
||||
);
|
||||
|
||||
$this->assertSame('project-test', $key->getProjectId());
|
||||
$this->assertContains('files.read', $key->getScopes());
|
||||
}
|
||||
|
||||
public function testVerifiesUnsignedPayloadSignature(): void
|
||||
{
|
||||
$secret = 'test-api-key-secret';
|
||||
$request = $this->signedRequest('GET', '/v1/s3/bucket', '', $secret, 'UNSIGNED-PAYLOAD', 'delimiter=%2F&list-type=2&max-keys=100');
|
||||
|
||||
$key = (new SignatureV4())->verify(
|
||||
$request,
|
||||
$this->project($secret, ['files.read']),
|
||||
new Document(),
|
||||
new User(),
|
||||
['files.read']
|
||||
);
|
||||
|
||||
$this->assertSame('project-test', $key->getProjectId());
|
||||
}
|
||||
|
||||
public function testRejectsExpiredPresignedRequest(): void
|
||||
{
|
||||
$secret = 'test-api-key-secret';
|
||||
$request = $this->presignedRequestAt(
|
||||
'GET',
|
||||
'/v1/s3/bucket/object.txt',
|
||||
$secret,
|
||||
\gmdate('Ymd\THis\Z', \time() - 120),
|
||||
60,
|
||||
);
|
||||
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('presigned URL has expired');
|
||||
|
||||
(new SignatureV4())->verify(
|
||||
$request,
|
||||
$this->project($secret, ['files.read']),
|
||||
new Document(),
|
||||
new User(),
|
||||
['files.read']
|
||||
);
|
||||
}
|
||||
public function testVerifiesOpenDalRangedGetSignature(): void
|
||||
{
|
||||
$secret = 'test-api-key-secret';
|
||||
$request = $this->signedRequest(
|
||||
'GET',
|
||||
'/v1/s3/test/ai_history_fundamentals_quiz_quizizz.xlsx',
|
||||
'',
|
||||
$secret,
|
||||
'UNSIGNED-PAYLOAD',
|
||||
additionalHeaders: ['range' => 'bytes=0-8218'],
|
||||
);
|
||||
|
||||
$key = (new SignatureV4())->verify(
|
||||
$request,
|
||||
$this->project($secret, ['files.read']),
|
||||
new Document(),
|
||||
new User(),
|
||||
['files.read']
|
||||
);
|
||||
|
||||
$this->assertSame('project-test', $key->getProjectId());
|
||||
}
|
||||
|
||||
public function testVerifiesOpenDalDeleteSignature(): void
|
||||
{
|
||||
$secret = 'test-api-key-secret';
|
||||
$request = $this->signedRequest(
|
||||
'DELETE',
|
||||
'/v1/s3/test/ai_history_fundamentals_quiz_quizizz.xlsx',
|
||||
'',
|
||||
$secret,
|
||||
'UNSIGNED-PAYLOAD',
|
||||
);
|
||||
|
||||
$key = (new SignatureV4())->verify(
|
||||
$request,
|
||||
$this->project($secret, ['files.write']),
|
||||
new Document(),
|
||||
new User(),
|
||||
['files.write']
|
||||
);
|
||||
|
||||
$this->assertSame('project-test', $key->getProjectId());
|
||||
}
|
||||
|
||||
public function testRejectsValidSignatureWhenApiKeyMissingRequiredScope(): void
|
||||
{
|
||||
$secret = 'test-api-key-secret';
|
||||
$request = $this->signedRequest('PUT', '/v1/s3/bucket/object.txt', 'body', $secret);
|
||||
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('API key is missing required S3 storage scopes.');
|
||||
|
||||
(new SignatureV4())->verify(
|
||||
$request,
|
||||
$this->project($secret, ['files.read']),
|
||||
new Document(),
|
||||
new User(),
|
||||
['files.write']
|
||||
);
|
||||
}
|
||||
|
||||
public function testRejectsInvalidSignature(): void
|
||||
{
|
||||
$secret = 'test-api-key-secret';
|
||||
$request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret);
|
||||
$request->headerMap['authorization'] = \preg_replace('/Signature=[0-9a-f]+/', 'Signature=deadbeef', $request->headerMap['authorization']);
|
||||
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('Signature does not match.');
|
||||
|
||||
(new SignatureV4())->verify(
|
||||
$request,
|
||||
$this->project($secret, ['files.read']),
|
||||
new Document(),
|
||||
new User(),
|
||||
['files.read']
|
||||
);
|
||||
}
|
||||
|
||||
public function testRejectsExpiredSignatureTimestamp(): void
|
||||
{
|
||||
$secret = 'test-api-key-secret';
|
||||
$date = \gmdate('Ymd\THis\Z', \time() - 901);
|
||||
$request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret, date: $date);
|
||||
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('AWS Signature V4 timestamp is outside the allowed time window.');
|
||||
|
||||
(new SignatureV4())->verify(
|
||||
$request,
|
||||
$this->project($secret, ['files.read']),
|
||||
new Document(),
|
||||
new User(),
|
||||
['files.read']
|
||||
);
|
||||
}
|
||||
|
||||
public function testRejectsFutureSignatureTimestamp(): void
|
||||
{
|
||||
$secret = 'test-api-key-secret';
|
||||
$date = \gmdate('Ymd\THis\Z', \time() + 901);
|
||||
$request = $this->signedRequest('GET', '/v1/s3/bucket/object.txt', '', $secret, date: $date);
|
||||
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('AWS Signature V4 timestamp is outside the allowed time window.');
|
||||
|
||||
(new SignatureV4())->verify(
|
||||
$request,
|
||||
$this->project($secret, ['files.read']),
|
||||
new Document(),
|
||||
new User(),
|
||||
['files.read']
|
||||
);
|
||||
}
|
||||
|
||||
private function project(string $secret, array $scopes): Document
|
||||
{
|
||||
return new Document([
|
||||
'$id' => 'project-test',
|
||||
'keys' => [
|
||||
[
|
||||
'$id' => 'key-test',
|
||||
'name' => 'S3 Test Key',
|
||||
'secret' => $secret,
|
||||
'scopes' => $scopes,
|
||||
'expire' => null,
|
||||
],
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
private function presignedRequestAt(string $method, string $uri, string $secret, string $date, int $expires): TestRequest
|
||||
{
|
||||
$shortDate = \substr($date, 0, 8);
|
||||
$scope = "{$shortDate}/us-east-1/s3/aws4_request";
|
||||
$parameters = [
|
||||
'X-Amz-Algorithm' => 'AWS4-HMAC-SHA256',
|
||||
'X-Amz-Credential' => 'project-test/' . $scope,
|
||||
'X-Amz-Date' => $date,
|
||||
'X-Amz-Expires' => (string) $expires,
|
||||
'X-Amz-SignedHeaders' => 'host',
|
||||
];
|
||||
$encoded = [];
|
||||
foreach ($parameters as $name => $value) {
|
||||
$encoded[] = \rawurlencode($name) . '=' . \rawurlencode($value);
|
||||
}
|
||||
\sort($encoded, SORT_STRING);
|
||||
$canonicalQuery = \implode('&', $encoded);
|
||||
$canonicalRequest = \implode("\n", [
|
||||
$method,
|
||||
$uri,
|
||||
$canonicalQuery,
|
||||
"host:cloud.appwrite.test\n",
|
||||
'host',
|
||||
'UNSIGNED-PAYLOAD',
|
||||
]);
|
||||
$stringToSign = "AWS4-HMAC-SHA256\n{$date}\n{$scope}\n" . \hash('sha256', $canonicalRequest);
|
||||
$signature = \hash_hmac('sha256', $stringToSign, $this->signingKey($secret, $shortDate, 'us-east-1', 's3'));
|
||||
$query = $canonicalQuery . '&X-Amz-Signature=' . $signature;
|
||||
|
||||
return new TestRequest($method, $uri . '?' . $query, ['host' => 'cloud.appwrite.test'], '', $query);
|
||||
}
|
||||
|
||||
private function signedRequest(string $method, string $uri, string $body, string $secret, ?string $payloadHash = null, string $query = '', ?string $date = null, array $additionalHeaders = []): TestRequest
|
||||
{
|
||||
$date ??= \gmdate('Ymd\THis\Z');
|
||||
$shortDate = \substr($date, 0, 8);
|
||||
$region = 'us-east-1';
|
||||
$service = 's3';
|
||||
$headers = \array_merge([
|
||||
'host' => 'cloud.appwrite.test',
|
||||
'x-amz-content-sha256' => $payloadHash ?? \hash('sha256', $body),
|
||||
'x-amz-date' => $date,
|
||||
], $additionalHeaders);
|
||||
\ksort($headers);
|
||||
|
||||
$signedHeaders = \implode(';', \array_keys($headers));
|
||||
$canonicalHeaders = '';
|
||||
foreach ($headers as $name => $value) {
|
||||
$canonicalHeaders .= $name . ':' . \preg_replace('/\s+/', ' ', \trim($value)) . "\n";
|
||||
}
|
||||
$canonicalRequest = \implode("\n", [
|
||||
$method,
|
||||
$uri,
|
||||
$query,
|
||||
$canonicalHeaders,
|
||||
$signedHeaders,
|
||||
$headers['x-amz-content-sha256'],
|
||||
]);
|
||||
|
||||
$scope = "{$shortDate}/{$region}/{$service}/aws4_request";
|
||||
$stringToSign = "AWS4-HMAC-SHA256\n{$date}\n{$scope}\n" . \hash('sha256', $canonicalRequest);
|
||||
$signature = \hash_hmac('sha256', $stringToSign, $this->signingKey($secret, $shortDate, $region, $service));
|
||||
|
||||
$headers['authorization'] = "AWS4-HMAC-SHA256 Credential=project-test/{$scope}, SignedHeaders={$signedHeaders}, Signature={$signature}";
|
||||
|
||||
return new TestRequest($method, $query === '' ? $uri : $uri . '?' . $query, $headers, $body, $query);
|
||||
}
|
||||
|
||||
private function signingKey(string $secret, string $date, string $region, string $service): string
|
||||
{
|
||||
$kDate = \hash_hmac('sha256', $date, 'AWS4' . $secret, true);
|
||||
$kRegion = \hash_hmac('sha256', $region, $kDate, true);
|
||||
$kService = \hash_hmac('sha256', $service, $kRegion, true);
|
||||
return \hash_hmac('sha256', 'aws4_request', $kService, true);
|
||||
}
|
||||
}
|
||||
|
||||
final class TestRequest extends Request
|
||||
{
|
||||
/**
|
||||
* @param array<string, string|null> $headerMap
|
||||
*/
|
||||
public function __construct(
|
||||
private readonly string $method,
|
||||
public readonly string $uri,
|
||||
public array $headerMap,
|
||||
private readonly string $body,
|
||||
public readonly string $query = '',
|
||||
) {
|
||||
}
|
||||
|
||||
public function getMethod(): string
|
||||
{
|
||||
return $this->method;
|
||||
}
|
||||
|
||||
public function getURI(): string
|
||||
{
|
||||
return $this->uri;
|
||||
}
|
||||
|
||||
public function getHeaderLine(string $key, string $default = ''): string
|
||||
{
|
||||
return $this->headerMap[\strtolower($key)] ?? $default;
|
||||
}
|
||||
|
||||
public function getRawPayload(): string
|
||||
{
|
||||
return $this->body;
|
||||
}
|
||||
|
||||
public function getServer(string $key, ?string $default = null): ?string
|
||||
{
|
||||
return $key === 'query_string' ? $this->query : $default;
|
||||
}
|
||||
|
||||
public function getSwooleRequest(): SwooleRequest
|
||||
{
|
||||
throw new \LogicException('Not used by this test.');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Platform\Modules\S3\Requests;
|
||||
|
||||
use Appwrite\Extend\Exception as AppwriteException;
|
||||
use Appwrite\Platform\Modules\S3\Requests\AwsChunked;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
final class AwsChunkedTest extends TestCase
|
||||
{
|
||||
public function testAppliesDetectsStreamingPayloadsAndChunkedEncoding(): void
|
||||
{
|
||||
$this->assertTrue(AwsChunked::applies('STREAMING-UNSIGNED-PAYLOAD-TRAILER', ''));
|
||||
$this->assertTrue(AwsChunked::applies('streaming-aws4-hmac-sha256-payload', ''));
|
||||
$this->assertTrue(AwsChunked::applies('', 'aws-chunked'));
|
||||
$this->assertTrue(AwsChunked::applies('', 'gzip, aws-chunked'));
|
||||
|
||||
$this->assertFalse(AwsChunked::applies(\hash('sha256', 'body'), ''));
|
||||
$this->assertFalse(AwsChunked::applies('UNSIGNED-PAYLOAD', ''));
|
||||
$this->assertFalse(AwsChunked::applies('', 'gzip'));
|
||||
}
|
||||
|
||||
public function testDecodesUnsignedTrailerFraming(): void
|
||||
{
|
||||
$payload = "6\r\nhello \r\n"
|
||||
. "5\r\nworld\r\n"
|
||||
. "0\r\n"
|
||||
. 'x-amz-checksum-crc32:' . \base64_encode(\hash('crc32b', 'hello world', true)) . "\r\n"
|
||||
. "\r\n";
|
||||
|
||||
$this->assertSame('hello world', AwsChunked::decode($payload, 11));
|
||||
}
|
||||
|
||||
public function testDecodesSignedChunkFraming(): void
|
||||
{
|
||||
$payload = "6;chunk-signature=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\nhello \r\n"
|
||||
. "5;chunk-signature=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\r\nworld\r\n"
|
||||
. "0;chunk-signature=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\r\n\r\n";
|
||||
|
||||
$this->assertSame('hello world', AwsChunked::decode($payload, 11));
|
||||
}
|
||||
|
||||
public function testDecodesEmptyBody(): void
|
||||
{
|
||||
$this->assertSame('', AwsChunked::decode("0\r\n\r\n", 0));
|
||||
$this->assertSame('', AwsChunked::decode("0\r\n", null));
|
||||
}
|
||||
|
||||
public function testDecodeIgnoresUnverifiedTrailerAlgorithms(): void
|
||||
{
|
||||
$payload = "3\r\nabc\r\n0\r\nx-amz-checksum-sha256:not-checked\r\n\r\n";
|
||||
|
||||
$this->assertSame('abc', AwsChunked::decode($payload, 3));
|
||||
}
|
||||
|
||||
public function testRejectsDeclaredLengthMismatch(): void
|
||||
{
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('x-amz-decoded-content-length');
|
||||
|
||||
AwsChunked::decode("3\r\nabc\r\n0\r\n\r\n", 4);
|
||||
}
|
||||
|
||||
public function testRejectsCrc32TrailerMismatch(): void
|
||||
{
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('CRC32');
|
||||
|
||||
AwsChunked::decode("3\r\nabc\r\n0\r\n" . 'x-amz-checksum-crc32:' . \base64_encode(\hash('crc32b', 'abd', true)) . "\r\n\r\n", 3);
|
||||
}
|
||||
|
||||
public function testRejectsTruncatedChunkData(): void
|
||||
{
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('truncated');
|
||||
|
||||
AwsChunked::decode("6\r\nhel", null);
|
||||
}
|
||||
|
||||
public function testRejectsMissingFinalChunk(): void
|
||||
{
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('zero-size chunk');
|
||||
|
||||
AwsChunked::decode("3\r\nabc\r\n", null);
|
||||
}
|
||||
|
||||
public function testRejectsNonHexadecimalChunkSize(): void
|
||||
{
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('hexadecimal');
|
||||
|
||||
AwsChunked::decode("zz\r\nabc\r\n0\r\n\r\n", null);
|
||||
}
|
||||
|
||||
public function testRejectsChunkDataWithoutCrlfTerminator(): void
|
||||
{
|
||||
$this->expectException(AppwriteException::class);
|
||||
$this->expectExceptionMessage('CRLF');
|
||||
|
||||
AwsChunked::decode("3\r\nabcXX0\r\n\r\n", null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Tests\Unit\Platform\Modules\S3\Responses;
|
||||
|
||||
use Appwrite\Platform\Modules\S3\Responses\S3Xml;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Utopia\Database\Document;
|
||||
|
||||
final class S3XmlTest extends TestCase
|
||||
{
|
||||
public function testListBucketsReturnsS3Xml(): void
|
||||
{
|
||||
$xml = S3Xml::listBuckets([
|
||||
new Document([
|
||||
'$id' => 'bucket-a',
|
||||
'$createdAt' => '2026-01-01T00:00:00.000+00:00',
|
||||
]),
|
||||
]);
|
||||
|
||||
$this->assertStringStartsWith('<?xml version="1.0" encoding="UTF-8"?>', $xml);
|
||||
$this->assertStringContainsString('<ListAllMyBucketsResult>', $xml);
|
||||
$this->assertStringContainsString('<Name>bucket-a</Name>', $xml);
|
||||
}
|
||||
|
||||
public function testListObjectsEscapesKeys(): void
|
||||
{
|
||||
$xml = S3Xml::listObjects('bucket-a', [
|
||||
new Document([
|
||||
'$id' => 'filea',
|
||||
'$createdAt' => '2026-01-01T00:00:00.000+00:00',
|
||||
'name' => 'folder/file&one.txt',
|
||||
'signature' => 'etag-a',
|
||||
'sizeOriginal' => 12,
|
||||
]),
|
||||
]);
|
||||
|
||||
$this->assertStringContainsString('<Key>folder/file&one.txt</Key>', $xml);
|
||||
}
|
||||
|
||||
public function testListObjectsUsesFolderAndName(): void
|
||||
{
|
||||
$xml = S3Xml::listObjects('bucket-a', [
|
||||
$this->file('filea', 'file&one.txt', 'folder/'),
|
||||
]);
|
||||
|
||||
$this->assertStringContainsString('<Key>folder/file&one.txt</Key>', $xml);
|
||||
}
|
||||
|
||||
public function testListObjectsSupportsDelimiterAndMarkerPagination(): void
|
||||
{
|
||||
$firstPage = S3Xml::listObjects('bucket-a', [
|
||||
$this->file('filea', 'readme.txt', 'docs/'),
|
||||
$this->file('fileb', 'a.txt', 'photos/2025/'),
|
||||
$this->file('filec', 'b.txt', 'photos/2026/'),
|
||||
], delimiter: '/', maxKeys: 1);
|
||||
|
||||
$this->assertStringContainsString('<CommonPrefixes><Prefix>docs/</Prefix></CommonPrefixes>', $firstPage);
|
||||
$this->assertStringContainsString('<IsTruncated>true</IsTruncated>', $firstPage);
|
||||
$this->assertStringContainsString('<NextMarker>docs/</NextMarker>', $firstPage);
|
||||
|
||||
$secondPage = S3Xml::listObjects('bucket-a', [
|
||||
$this->file('filea', 'readme.txt', 'docs/'),
|
||||
$this->file('fileb', 'a.txt', 'photos/2025/'),
|
||||
$this->file('filec', 'b.txt', 'photos/2026/'),
|
||||
], delimiter: '/', maxKeys: 1, marker: 'docs/');
|
||||
|
||||
$this->assertStringNotContainsString('<Prefix>docs/</Prefix>', $secondPage);
|
||||
$this->assertStringContainsString('<CommonPrefixes><Prefix>photos/</Prefix></CommonPrefixes>', $secondPage);
|
||||
$this->assertStringContainsString('<IsTruncated>false</IsTruncated>', $secondPage);
|
||||
}
|
||||
|
||||
public function testListObjectsV2SupportsPrefixPaginationAndDelimiter(): void
|
||||
{
|
||||
$xml = S3Xml::listObjectsV2('bucket-a', [
|
||||
$this->file('filea', 'readme.txt', 'docs/'),
|
||||
$this->file('fileb', 'c.txt', 'photos/2025/'),
|
||||
$this->file('filec', 'a.txt', 'photos/2026/'),
|
||||
$this->file('filed', 'b.txt', 'photos/2026/'),
|
||||
], prefix: 'photos/', delimiter: '/', maxKeys: 1);
|
||||
|
||||
$this->assertStringContainsString('<ListBucketResult>', $xml);
|
||||
$this->assertStringContainsString('<Prefix>photos/</Prefix>', $xml);
|
||||
$this->assertStringContainsString('<KeyCount>1</KeyCount>', $xml);
|
||||
$this->assertStringContainsString('<MaxKeys>1</MaxKeys>', $xml);
|
||||
$this->assertStringContainsString('<IsTruncated>true</IsTruncated>', $xml);
|
||||
$this->assertStringContainsString('<CommonPrefixes><Prefix>photos/2025/</Prefix></CommonPrefixes>', $xml);
|
||||
$this->assertStringContainsString('<NextContinuationToken>', $xml);
|
||||
}
|
||||
|
||||
public function testListObjectsV2ContinuationTokenStartsAfterLastKey(): void
|
||||
{
|
||||
$firstPage = S3Xml::listObjectsV2('bucket-a', [
|
||||
$this->file('filea', 'a.txt'),
|
||||
$this->file('fileb', 'b.txt'),
|
||||
], maxKeys: 1);
|
||||
|
||||
preg_match('/<NextContinuationToken>([^<]+)<\/NextContinuationToken>/', $firstPage, $matches);
|
||||
$this->assertNotEmpty($matches[1] ?? '');
|
||||
|
||||
$secondPage = S3Xml::listObjectsV2('bucket-a', [
|
||||
$this->file('filea', 'a.txt'),
|
||||
$this->file('fileb', 'b.txt'),
|
||||
], continuationToken: $matches[1]);
|
||||
|
||||
$this->assertStringNotContainsString('<Key>a.txt</Key>', $secondPage);
|
||||
$this->assertStringContainsString('<Key>b.txt</Key>', $secondPage);
|
||||
$this->assertStringContainsString('<IsTruncated>false</IsTruncated>', $secondPage);
|
||||
}
|
||||
|
||||
public function testListObjectsV2OrdersAndPaginatesNumericKeysLexicographically(): void
|
||||
{
|
||||
$files = [
|
||||
$this->file('filea', '9'),
|
||||
$this->file('fileb', '10'),
|
||||
];
|
||||
|
||||
$firstPage = S3Xml::listObjectsV2('bucket-a', $files, maxKeys: 1);
|
||||
$this->assertStringContainsString('<Key>10</Key>', $firstPage);
|
||||
$this->assertStringNotContainsString('<Key>9</Key>', $firstPage);
|
||||
$this->assertStringContainsString('<IsTruncated>true</IsTruncated>', $firstPage);
|
||||
|
||||
preg_match('/<NextContinuationToken>([^<]+)<\/NextContinuationToken>/', $firstPage, $matches);
|
||||
$this->assertNotEmpty($matches[1] ?? '');
|
||||
|
||||
$secondPage = S3Xml::listObjectsV2('bucket-a', $files, maxKeys: 1, continuationToken: $matches[1]);
|
||||
$this->assertStringContainsString('<Key>9</Key>', $secondPage);
|
||||
$this->assertStringContainsString('<IsTruncated>false</IsTruncated>', $secondPage);
|
||||
}
|
||||
|
||||
public function testMultipartResponses(): void
|
||||
{
|
||||
$init = S3Xml::initiateMultipart('bucket-a', 'object.txt', 'upload-id');
|
||||
$parts = S3Xml::listParts('bucket-a', 'object.txt', 'upload-id', [
|
||||
['partNumber' => 1, 'etag' => 'etag-1', 'size' => 10],
|
||||
]);
|
||||
$complete = S3Xml::completeMultipart('bucket-a', 'object.txt', 'final-etag');
|
||||
|
||||
$this->assertStringContainsString('<UploadId>upload-id</UploadId>', $init);
|
||||
$this->assertStringContainsString('<PartNumber>1</PartNumber>', $parts);
|
||||
$this->assertStringContainsString('<ETag>"final-etag"</ETag>', $complete);
|
||||
}
|
||||
|
||||
public function testAccessControlPolicyRendersNestedGrantee(): void
|
||||
{
|
||||
$xml = S3Xml::accessControlPolicy();
|
||||
|
||||
$this->assertStringContainsString('<Owner><ID>appwrite</ID><DisplayName>appwrite</DisplayName></Owner>', $xml);
|
||||
$this->assertStringContainsString('<Grantee><ID>appwrite</ID><DisplayName>appwrite</DisplayName></Grantee>', $xml);
|
||||
$this->assertStringContainsString('<Permission>FULL_CONTROL</Permission>', $xml);
|
||||
$this->assertStringNotContainsString('<Grantee>Array</Grantee>', $xml);
|
||||
}
|
||||
|
||||
private function file(string $id, string $name, string $folder = ''): Document
|
||||
{
|
||||
return new Document([
|
||||
'$id' => $id,
|
||||
'$createdAt' => '2026-01-01T00:00:00.000+00:00',
|
||||
'name' => $name,
|
||||
'folder' => $folder,
|
||||
'signature' => 'etag-' . $id,
|
||||
'sizeOriginal' => 12,
|
||||
]);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user