mirror of
https://github.com/appwrite/appwrite.git
synced 2026-11-18 20:36:03 +00:00
Fix parallel deployment chunk uploads on S3 devices (#12574)
This commit is contained in:
1 parent
36ab95f1c8
commit
9c90f4040b
2 files changed
+156
-107
No files matched your search
@@ -202,8 +202,27 @@ class Create extends Action
|
||||
$metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)];
|
||||
$completed = false;
|
||||
|
||||
$mergeUploadMetadata = function (array $stored, array $current): array {
|
||||
$merged = \array_merge($stored, $current);
|
||||
|
||||
if (isset($stored['parts']) || isset($current['parts'])) {
|
||||
$parts = $stored['parts'] ?? [];
|
||||
foreach (($current['parts'] ?? []) as $part => $value) {
|
||||
$parts[(int) $part] = $value;
|
||||
}
|
||||
\ksort($parts);
|
||||
|
||||
$merged['parts'] = $parts;
|
||||
$merged['chunks'] = \count($parts);
|
||||
}
|
||||
|
||||
return $merged;
|
||||
};
|
||||
|
||||
$type = $request->getHeaderLine('x-sdk-language') === 'cli' ? 'cli' : 'manual';
|
||||
|
||||
try {
|
||||
$locks($lockKey, 600, function () use (&$chunks, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void {
|
||||
$locks($lockKey, 600, function () use ($activate, &$chunks, $commands, $contentRange, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, &$metadata, $path, $type, &$completed, $response): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
|
||||
if (!$deployment->isEmpty()) {
|
||||
@@ -220,6 +239,35 @@ class Create extends Action
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if ($deployment->isEmpty()) {
|
||||
$deviceForFunctions->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata);
|
||||
|
||||
if (!empty($contentRange)) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $function->getSequence(),
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceType' => 'functions',
|
||||
'entrypoint' => $entrypoint,
|
||||
'buildCommands' => $commands,
|
||||
'startCommand' => $function->getAttribute('startCommand', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => 0,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type
|
||||
]));
|
||||
}
|
||||
}
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
$response->addHeader('Retry-After', '5');
|
||||
@@ -231,23 +279,21 @@ class Create extends Action
|
||||
return;
|
||||
}
|
||||
|
||||
$chunksUploaded = $deviceForFunctions->upload($fileTmpName, $path, $chunk, $chunks, $metadata);
|
||||
$chunksUploaded = $deviceForFunctions->uploadChunk($fileTmpName, $path, $chunk, $chunks, $metadata);
|
||||
|
||||
if (empty($chunksUploaded)) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed moving file');
|
||||
}
|
||||
|
||||
$type = $request->getHeaderLine('x-sdk-language') === 'cli' ? 'cli' : 'manual';
|
||||
|
||||
try {
|
||||
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $platform, $project, $publisherForBuilds, $queueForEvents, $response, $type): void {
|
||||
$locks($lockKey, 600, function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $mergeUploadMetadata, $platform, $project, $publisherForBuilds, $queueForEvents, $response, $type): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
$uploaded = 0;
|
||||
|
||||
if (!$deployment->isEmpty()) {
|
||||
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
|
||||
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
|
||||
$metadata = \array_merge($deployment->getAttribute('sourceMetadata', []), $metadata);
|
||||
$metadata = $mergeUploadMetadata($deployment->getAttribute('sourceMetadata', []), $metadata);
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
$queueForEvents->reset();
|
||||
@@ -259,9 +305,11 @@ class Create extends Action
|
||||
}
|
||||
}
|
||||
|
||||
$chunksUploaded = max($uploaded, $chunksUploaded);
|
||||
$chunksUploaded = max($uploaded, $chunksUploaded, (int) ($metadata['chunks'] ?? 0));
|
||||
|
||||
if ($chunksUploaded === $chunks && $uploaded < $chunks) {
|
||||
$deviceForFunctions->finalizeUpload($path, $chunks, $metadata);
|
||||
|
||||
if ($activate) {
|
||||
// Remove deploy for all other deployments.
|
||||
$activeDeployments = $dbForProject->find('deployments', [
|
||||
@@ -320,36 +368,10 @@ class Create extends Action
|
||||
platform: $platform,
|
||||
));
|
||||
} else {
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $function->getSequence(),
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceType' => 'functions',
|
||||
'entrypoint' => $entrypoint,
|
||||
'buildCommands' => $commands,
|
||||
'startCommand' => $function->getAttribute('startCommand', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type
|
||||
]));
|
||||
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
|
||||
$metadata = null;
|
||||
|
||||
@@ -202,8 +202,35 @@ class Create extends Action
|
||||
$metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)];
|
||||
$completed = false;
|
||||
|
||||
$mergeUploadMetadata = function (array $stored, array $current): array {
|
||||
$merged = \array_merge($stored, $current);
|
||||
|
||||
if (isset($stored['parts']) || isset($current['parts'])) {
|
||||
$parts = $stored['parts'] ?? [];
|
||||
foreach (($current['parts'] ?? []) as $part => $value) {
|
||||
$parts[(int) $part] = $value;
|
||||
}
|
||||
\ksort($parts);
|
||||
|
||||
$merged['parts'] = $parts;
|
||||
$merged['chunks'] = \count($parts);
|
||||
}
|
||||
|
||||
return $merged;
|
||||
};
|
||||
|
||||
$type = $request->getHeaderLine('x-sdk-language') === 'cli' ? 'cli' : 'manual';
|
||||
|
||||
$commands = [];
|
||||
if (!empty($installCommand)) {
|
||||
$commands[] = $installCommand;
|
||||
}
|
||||
if (!empty($buildCommand)) {
|
||||
$commands[] = $buildCommand;
|
||||
}
|
||||
|
||||
try {
|
||||
$locks($lockKey, 600, function () use (&$chunks, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void {
|
||||
$locks($lockKey, 600, function () use ($activate, $authorization, &$chunks, $commands, $contentRange, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, &$site, $type, &$completed, $response): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
|
||||
if (!$deployment->isEmpty()) {
|
||||
@@ -220,6 +247,65 @@ class Create extends Action
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if ($deployment->isEmpty()) {
|
||||
$deviceForSites->prepareUpload($path, $metadata['content_type'] ?? '', $chunks, $metadata);
|
||||
|
||||
if (!empty($contentRange)) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $site->getSequence(),
|
||||
'resourceId' => $site->getId(),
|
||||
'resourceType' => 'sites',
|
||||
'buildCommands' => \implode(' && ', $commands),
|
||||
'startCommand' => $site->getAttribute('startCommand', ''),
|
||||
'buildOutput' => $outputDirectory,
|
||||
'adapter' => $site->getAttribute('adapter', ''),
|
||||
'fallbackFile' => $site->getAttribute('fallbackFile', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => 0,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type,
|
||||
]));
|
||||
|
||||
$sitesDomain = $platform['sitesDomain'];
|
||||
$domain = ID::unique() . "." . $sitesDomain;
|
||||
|
||||
// TODO: (@Meldiron) Remove after 1.7.x migration
|
||||
$isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5';
|
||||
$ruleId = $isMd5 ? md5($domain) : ID::unique();
|
||||
|
||||
$authorization->skip(
|
||||
fn () => $dbForPlatform->createDocument('rules', new Document([
|
||||
'$id' => $ruleId,
|
||||
'projectId' => $project->getId(),
|
||||
'projectInternalId' => $project->getSequence(),
|
||||
'domain' => $domain,
|
||||
'type' => 'deployment',
|
||||
'trigger' => 'deployment',
|
||||
'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(),
|
||||
'deploymentResourceType' => 'site',
|
||||
'deploymentResourceId' => $site->getId(),
|
||||
'deploymentResourceInternalId' => $site->getSequence(),
|
||||
'status' => 'verified',
|
||||
'certificateId' => '',
|
||||
'search' => implode(' ', [$ruleId, $domain]),
|
||||
'owner' => 'Appwrite',
|
||||
'region' => $project->getAttribute('region')
|
||||
]))
|
||||
);
|
||||
}
|
||||
}
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
$response->addHeader('Retry-After', '5');
|
||||
@@ -231,31 +317,21 @@ class Create extends Action
|
||||
return;
|
||||
}
|
||||
|
||||
$chunksUploaded = $deviceForSites->upload($fileTmpName, $path, $chunk, $chunks, $metadata);
|
||||
$chunksUploaded = $deviceForSites->uploadChunk($fileTmpName, $path, $chunk, $chunks, $metadata);
|
||||
|
||||
if (empty($chunksUploaded)) {
|
||||
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed moving file');
|
||||
}
|
||||
|
||||
$type = $request->getHeaderLine('x-sdk-language') === 'cli' ? 'cli' : 'manual';
|
||||
|
||||
$commands = [];
|
||||
if (!empty($installCommand)) {
|
||||
$commands[] = $installCommand;
|
||||
}
|
||||
if (!empty($buildCommand)) {
|
||||
$commands[] = $buildCommand;
|
||||
}
|
||||
|
||||
try {
|
||||
$locks($lockKey, 600, function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, $publisherForBuilds, $queueForEvents, $response, &$site, $siteId, $type): void {
|
||||
$locks($lockKey, 600, function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $mergeUploadMetadata, $outputDirectory, $path, $platform, $project, $publisherForBuilds, $queueForEvents, $response, &$site, $siteId, $type): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
$uploaded = 0;
|
||||
|
||||
if (!$deployment->isEmpty()) {
|
||||
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
|
||||
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
|
||||
$metadata = \array_merge($deployment->getAttribute('sourceMetadata', []), $metadata);
|
||||
$metadata = $mergeUploadMetadata($deployment->getAttribute('sourceMetadata', []), $metadata);
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
$queueForEvents->reset();
|
||||
@@ -267,9 +343,11 @@ class Create extends Action
|
||||
}
|
||||
}
|
||||
|
||||
$chunksUploaded = max($uploaded, $chunksUploaded);
|
||||
$chunksUploaded = max($uploaded, $chunksUploaded, (int) ($metadata['chunks'] ?? 0));
|
||||
|
||||
if ($chunksUploaded === $chunks && $uploaded < $chunks) {
|
||||
$deviceForSites->finalizeUpload($path, $chunks, $metadata);
|
||||
|
||||
if ($activate) {
|
||||
// Remove deploy for all other deployments.
|
||||
$activeDeployments = $dbForProject->find('deployments', [
|
||||
@@ -355,61 +433,10 @@ class Create extends Action
|
||||
platform: $platform,
|
||||
));
|
||||
} else {
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $site->getSequence(),
|
||||
'resourceId' => $site->getId(),
|
||||
'resourceType' => 'sites',
|
||||
'buildCommands' => \implode(' && ', $commands),
|
||||
'startCommand' => $site->getAttribute('startCommand', ''),
|
||||
'buildOutput' => $outputDirectory,
|
||||
'adapter' => $site->getAttribute('adapter', ''),
|
||||
'fallbackFile' => $site->getAttribute('fallbackFile', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type,
|
||||
]));
|
||||
|
||||
$sitesDomain = $platform['sitesDomain'];
|
||||
$domain = ID::unique() . "." . $sitesDomain;
|
||||
$ruleId = md5($domain);
|
||||
$authorization->skip(
|
||||
fn () => $dbForPlatform->createDocument('rules', new Document([
|
||||
'$id' => $ruleId,
|
||||
'projectId' => $project->getId(),
|
||||
'projectInternalId' => $project->getSequence(),
|
||||
'domain' => $domain,
|
||||
'type' => 'deployment',
|
||||
'trigger' => 'deployment',
|
||||
'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(),
|
||||
'deploymentResourceType' => 'site',
|
||||
'deploymentResourceId' => $site->getId(),
|
||||
'deploymentResourceInternalId' => $site->getSequence(),
|
||||
'status' => 'verified',
|
||||
'certificateId' => '',
|
||||
'search' => implode(' ', [$ruleId, $domain]),
|
||||
'owner' => 'Appwrite',
|
||||
'region' => $project->getAttribute('region')
|
||||
]))
|
||||
);
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
|
||||
$metadata = null;
|
||||
|
||||
Reference in new issue
Block a user