mirror of
https://github.com/appwrite/appwrite.git
synced 2026-05-26 13:51:13 +00:00
Fix parallel deployment chunk uploads
This commit is contained in:
Torsten Dittmann
@@ -20,6 +20,8 @@ use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Lock\Distributed;
|
||||
use Utopia\Lock\Exception\Contention as LockContention;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Storage\Device;
|
||||
@@ -90,6 +92,7 @@ class Create extends Action
|
||||
->inject('queueForBuilds')
|
||||
->inject('plan')
|
||||
->inject('authorization')
|
||||
->inject('redis')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
@@ -108,7 +111,8 @@ class Create extends Action
|
||||
Device $deviceForLocal,
|
||||
Build $queueForBuilds,
|
||||
array $plan,
|
||||
Authorization $authorization
|
||||
Authorization $authorization,
|
||||
\Redis $redis
|
||||
) {
|
||||
$activate = \strval($activate) === 'true' || \strval($activate) === '1';
|
||||
|
||||
@@ -190,20 +194,40 @@ class Create extends Action
|
||||
// Save to storage
|
||||
$fileSize ??= $deviceForLocal->getFileSize($fileTmpName);
|
||||
$path = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
|
||||
$lockKey = 'functions:deployment:' . $project->getId() . ':' . $functionId . ':' . $deploymentId;
|
||||
$checkLock = new Distributed($redis, $lockKey, ttl: 120);
|
||||
$stateLock = new Distributed($redis, $lockKey, ttl: 600);
|
||||
|
||||
$metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)];
|
||||
if (!$deployment->isEmpty()) {
|
||||
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
|
||||
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
|
||||
$metadata = $deployment->getAttribute('sourceMetadata', []);
|
||||
$completed = false;
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
$checkLock->withLock(function () use (&$chunks, $contentRange, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
|
||||
if (!$deployment->isEmpty()) {
|
||||
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
|
||||
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
|
||||
$metadata = $deployment->getAttribute('sourceMetadata', []);
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
|
||||
$completed = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
$response->addHeader('Retry-After', '5');
|
||||
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.');
|
||||
}
|
||||
|
||||
if ($completed) {
|
||||
return;
|
||||
}
|
||||
|
||||
$chunksUploaded = $deviceForFunctions->upload($fileTmpName, $path, $chunk, $chunks, $metadata);
|
||||
@@ -214,115 +238,141 @@ class Create extends Action
|
||||
|
||||
$type = $request->getHeader('x-sdk-language') === 'cli' ? 'cli' : 'manual';
|
||||
|
||||
if ($chunksUploaded === $chunks) {
|
||||
if ($activate) {
|
||||
// Remove deploy for all other deployments.
|
||||
$activeDeployments = $dbForProject->find('deployments', [
|
||||
Query::equal('activate', [true]),
|
||||
Query::equal('resourceId', [$functionId]),
|
||||
Query::equal('resourceType', ['functions'])
|
||||
]);
|
||||
try {
|
||||
$stateLock->withLock(function () use ($activate, &$chunks, $chunksUploaded, $commands, $dbForProject, $deploymentId, $deviceForFunctions, $entrypoint, $fileSize, &$function, $functionId, $path, &$metadata, $queueForBuilds, $queueForEvents, $response, $type): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
$uploaded = 0;
|
||||
|
||||
foreach ($activeDeployments as $activeDeployment) {
|
||||
$activeDeployment->setAttribute('activate', false);
|
||||
$dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document([
|
||||
'activate' => false,
|
||||
]));
|
||||
if (!$deployment->isEmpty()) {
|
||||
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
|
||||
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
|
||||
$metadata = \array_merge($deployment->getAttribute('sourceMetadata', []), $metadata);
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$fileSize = $deviceForFunctions->getFileSize($path);
|
||||
$chunksUploaded = max($uploaded, $chunksUploaded);
|
||||
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $function->getSequence(),
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceType' => 'functions',
|
||||
'entrypoint' => $entrypoint,
|
||||
'buildCommands' => $commands,
|
||||
'startCommand' => $function->getAttribute('startCommand', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type
|
||||
]));
|
||||
if ($chunksUploaded === $chunks && $uploaded < $chunks) {
|
||||
if ($activate) {
|
||||
// Remove deploy for all other deployments.
|
||||
$activeDeployments = $dbForProject->find('deployments', [
|
||||
Query::equal('activate', [true]),
|
||||
Query::equal('resourceId', [$functionId]),
|
||||
Query::equal('resourceType', ['functions'])
|
||||
]);
|
||||
|
||||
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document([
|
||||
'latestDeploymentId' => $deployment->getId(),
|
||||
'latestDeploymentInternalId' => $deployment->getSequence(),
|
||||
'latestDeploymentCreatedAt' => $deployment->getCreatedAt(),
|
||||
'latestDeploymentStatus' => $deployment->getAttribute('status', ''),
|
||||
]));
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceSize' => $fileSize,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
foreach ($activeDeployments as $activeDeployment) {
|
||||
$dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document([
|
||||
'activate' => false,
|
||||
]));
|
||||
}
|
||||
}
|
||||
|
||||
// Start the build
|
||||
$queueForBuilds
|
||||
->setType(BUILD_TYPE_DEPLOYMENT)
|
||||
->setResource($function)
|
||||
->setDeployment($deployment);
|
||||
} else {
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $function->getSequence(),
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceType' => 'functions',
|
||||
'entrypoint' => $entrypoint,
|
||||
'buildCommands' => $commands,
|
||||
'startCommand' => $function->getAttribute('startCommand', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type
|
||||
]));
|
||||
$fileSize = $deviceForFunctions->getFileSize($path);
|
||||
|
||||
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document([
|
||||
'latestDeploymentId' => $deployment->getId(),
|
||||
'latestDeploymentInternalId' => $deployment->getSequence(),
|
||||
'latestDeploymentCreatedAt' => $deployment->getCreatedAt(),
|
||||
'latestDeploymentStatus' => $deployment->getAttribute('status', ''),
|
||||
]));
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $function->getSequence(),
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceType' => 'functions',
|
||||
'entrypoint' => $entrypoint,
|
||||
'buildCommands' => $commands,
|
||||
'startCommand' => $function->getAttribute('startCommand', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type
|
||||
]));
|
||||
|
||||
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document([
|
||||
'latestDeploymentId' => $deployment->getId(),
|
||||
'latestDeploymentInternalId' => $deployment->getSequence(),
|
||||
'latestDeploymentCreatedAt' => $deployment->getCreatedAt(),
|
||||
'latestDeploymentStatus' => $deployment->getAttribute('status', ''),
|
||||
]));
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceSize' => $fileSize,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
|
||||
// Start the build
|
||||
$queueForBuilds
|
||||
->setType(BUILD_TYPE_DEPLOYMENT)
|
||||
->setResource($function)
|
||||
->setDeployment($deployment);
|
||||
} else {
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $function->getSequence(),
|
||||
'resourceId' => $function->getId(),
|
||||
'resourceType' => 'functions',
|
||||
'entrypoint' => $entrypoint,
|
||||
'buildCommands' => $commands,
|
||||
'startCommand' => $function->getAttribute('startCommand', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type
|
||||
]));
|
||||
|
||||
$function = $dbForProject->updateDocument('functions', $function->getId(), new Document([
|
||||
'latestDeploymentId' => $deployment->getId(),
|
||||
'latestDeploymentInternalId' => $deployment->getSequence(),
|
||||
'latestDeploymentCreatedAt' => $deployment->getCreatedAt(),
|
||||
'latestDeploymentStatus' => $deployment->getAttribute('status', ''),
|
||||
]));
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
}
|
||||
|
||||
$metadata = null;
|
||||
|
||||
if ($chunksUploaded === $chunks) {
|
||||
$queueForEvents
|
||||
->setParam('functionId', $function->getId())
|
||||
->setParam('deploymentId', $deployment->getId());
|
||||
}
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
$response->addHeader('Retry-After', '5');
|
||||
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.');
|
||||
}
|
||||
|
||||
$metadata = null;
|
||||
|
||||
$queueForEvents
|
||||
->setParam('functionId', $function->getId())
|
||||
->setParam('deploymentId', $deployment->getId());
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ use Utopia\Database\Query;
|
||||
use Utopia\Database\Validator\Authorization;
|
||||
use Utopia\Database\Validator\UID;
|
||||
use Utopia\Http\Adapter\Swoole\Request;
|
||||
use Utopia\Lock\Distributed;
|
||||
use Utopia\Lock\Exception\Contention as LockContention;
|
||||
use Utopia\Platform\Action;
|
||||
use Utopia\Platform\Scope\HTTP;
|
||||
use Utopia\Storage\Device;
|
||||
@@ -89,6 +91,7 @@ class Create extends Action
|
||||
->inject('plan')
|
||||
->inject('authorization')
|
||||
->inject('platform')
|
||||
->inject('redis')
|
||||
->callback($this->action(...));
|
||||
}
|
||||
|
||||
@@ -111,6 +114,7 @@ class Create extends Action
|
||||
array $plan,
|
||||
Authorization $authorization,
|
||||
array $platform,
|
||||
\Redis $redis,
|
||||
) {
|
||||
$activate = \strval($activate) === 'true' || \strval($activate) === '1';
|
||||
|
||||
@@ -192,20 +196,40 @@ class Create extends Action
|
||||
// Save to storage
|
||||
$fileSize ??= $deviceForLocal->getFileSize($fileTmpName);
|
||||
$path = $deviceForSites->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
|
||||
$lockKey = 'sites:deployment:' . $project->getId() . ':' . $siteId . ':' . $deploymentId;
|
||||
$checkLock = new Distributed($redis, $lockKey, ttl: 120);
|
||||
$stateLock = new Distributed($redis, $lockKey, ttl: 600);
|
||||
|
||||
$metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)];
|
||||
if (!$deployment->isEmpty()) {
|
||||
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
|
||||
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
|
||||
$metadata = $deployment->getAttribute('sourceMetadata', []);
|
||||
$completed = false;
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
$checkLock->withLock(function () use (&$chunks, $dbForProject, $deploymentId, &$metadata, &$completed, $response): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
|
||||
if (!$deployment->isEmpty()) {
|
||||
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
|
||||
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
|
||||
$metadata = $deployment->getAttribute('sourceMetadata', []);
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
|
||||
$completed = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
$response->addHeader('Retry-After', '5');
|
||||
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.');
|
||||
}
|
||||
|
||||
if ($completed) {
|
||||
return;
|
||||
}
|
||||
|
||||
$chunksUploaded = $deviceForSites->upload($fileTmpName, $path, $chunk, $chunks, $metadata);
|
||||
@@ -224,181 +248,205 @@ class Create extends Action
|
||||
$commands[] = $buildCommand;
|
||||
}
|
||||
|
||||
if ($chunksUploaded === $chunks) {
|
||||
if ($activate) {
|
||||
// Remove deploy for all other deployments.
|
||||
$activeDeployments = $dbForProject->find('deployments', [
|
||||
Query::equal('activate', [true]),
|
||||
Query::equal('resourceId', [$siteId]),
|
||||
Query::equal('resourceType', ['sites'])
|
||||
]);
|
||||
try {
|
||||
$stateLock->withLock(function () use ($activate, $authorization, $commands, &$chunks, $chunksUploaded, $dbForPlatform, $dbForProject, $deploymentId, $deviceForSites, $fileSize, &$metadata, $outputDirectory, $path, $platform, $project, $queueForBuilds, $queueForEvents, $response, &$site, $siteId, $type): void {
|
||||
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
|
||||
$uploaded = 0;
|
||||
|
||||
foreach ($activeDeployments as $activeDeployment) {
|
||||
$activeDeployment->setAttribute('activate', false);
|
||||
$dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document(['activate' => false]));
|
||||
if (!$deployment->isEmpty()) {
|
||||
$chunks = $deployment->getAttribute('sourceChunksTotal', 1);
|
||||
$uploaded = $deployment->getAttribute('sourceChunksUploaded', 0);
|
||||
$metadata = \array_merge($deployment->getAttribute('sourceMetadata', []), $metadata);
|
||||
|
||||
if ($uploaded === $chunks) {
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$fileSize = $deviceForSites->getFileSize($path);
|
||||
$chunksUploaded = max($uploaded, $chunksUploaded);
|
||||
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $site->getSequence(),
|
||||
'resourceId' => $site->getId(),
|
||||
'resourceType' => 'sites',
|
||||
'buildCommands' => \implode(' && ', $commands),
|
||||
'startCommand' => $site->getAttribute('startCommand', ''),
|
||||
'buildOutput' => $outputDirectory,
|
||||
'adapter' => $site->getAttribute('adapter', ''),
|
||||
'fallbackFile' => $site->getAttribute('fallbackFile', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type,
|
||||
]));
|
||||
if ($chunksUploaded === $chunks && $uploaded < $chunks) {
|
||||
if ($activate) {
|
||||
// Remove deploy for all other deployments.
|
||||
$activeDeployments = $dbForProject->find('deployments', [
|
||||
Query::equal('activate', [true]),
|
||||
Query::equal('resourceId', [$siteId]),
|
||||
Query::equal('resourceType', ['sites'])
|
||||
]);
|
||||
|
||||
$site = $site
|
||||
->setAttribute('latestDeploymentId', $deployment->getId())
|
||||
->setAttribute('latestDeploymentInternalId', $deployment->getSequence())
|
||||
->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt())
|
||||
->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', ''));
|
||||
$dbForProject->updateDocument('sites', $site->getId(), new Document([
|
||||
'latestDeploymentId' => $deployment->getId(),
|
||||
'latestDeploymentInternalId' => $deployment->getSequence(),
|
||||
'latestDeploymentCreatedAt' => $deployment->getCreatedAt(),
|
||||
'latestDeploymentStatus' => $deployment->getAttribute('status', ''),
|
||||
]));
|
||||
foreach ($activeDeployments as $activeDeployment) {
|
||||
$dbForProject->updateDocument('deployments', $activeDeployment->getId(), new Document(['activate' => false]));
|
||||
}
|
||||
}
|
||||
|
||||
$sitesDomain = $platform['sitesDomain'];
|
||||
$domain = ID::unique() . "." . $sitesDomain;
|
||||
$fileSize = $deviceForSites->getFileSize($path);
|
||||
|
||||
// TODO: (@Meldiron) Remove after 1.7.x migration
|
||||
$isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5';
|
||||
$ruleId = $isMd5 ? md5($domain) : ID::unique();
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $site->getSequence(),
|
||||
'resourceId' => $site->getId(),
|
||||
'resourceType' => 'sites',
|
||||
'buildCommands' => \implode(' && ', $commands),
|
||||
'startCommand' => $site->getAttribute('startCommand', ''),
|
||||
'buildOutput' => $outputDirectory,
|
||||
'adapter' => $site->getAttribute('adapter', ''),
|
||||
'fallbackFile' => $site->getAttribute('fallbackFile', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type,
|
||||
]));
|
||||
|
||||
$authorization->skip(
|
||||
fn () => $dbForPlatform->createDocument('rules', new Document([
|
||||
'$id' => $ruleId,
|
||||
'projectId' => $project->getId(),
|
||||
'projectInternalId' => $project->getSequence(),
|
||||
'domain' => $domain,
|
||||
'type' => 'deployment',
|
||||
'trigger' => 'deployment',
|
||||
'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(),
|
||||
'deploymentResourceType' => 'site',
|
||||
'deploymentResourceId' => $site->getId(),
|
||||
'deploymentResourceInternalId' => $site->getSequence(),
|
||||
'status' => 'verified',
|
||||
'certificateId' => '',
|
||||
'search' => implode(' ', [$ruleId, $domain]),
|
||||
'owner' => 'Appwrite',
|
||||
'region' => $project->getAttribute('region')
|
||||
]))
|
||||
);
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceSize' => $fileSize,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
$site = $site
|
||||
->setAttribute('latestDeploymentId', $deployment->getId())
|
||||
->setAttribute('latestDeploymentInternalId', $deployment->getSequence())
|
||||
->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt())
|
||||
->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', ''));
|
||||
$dbForProject->updateDocument('sites', $site->getId(), new Document([
|
||||
'latestDeploymentId' => $deployment->getId(),
|
||||
'latestDeploymentInternalId' => $deployment->getSequence(),
|
||||
'latestDeploymentCreatedAt' => $deployment->getCreatedAt(),
|
||||
'latestDeploymentStatus' => $deployment->getAttribute('status', ''),
|
||||
]));
|
||||
|
||||
// Start the build
|
||||
$queueForBuilds
|
||||
->setType(BUILD_TYPE_DEPLOYMENT)
|
||||
->setResource($site)
|
||||
->setDeployment($deployment);
|
||||
} else {
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $site->getSequence(),
|
||||
'resourceId' => $site->getId(),
|
||||
'resourceType' => 'sites',
|
||||
'buildCommands' => \implode(' && ', $commands),
|
||||
'startCommand' => $site->getAttribute('startCommand', ''),
|
||||
'buildOutput' => $outputDirectory,
|
||||
'adapter' => $site->getAttribute('adapter', ''),
|
||||
'fallbackFile' => $site->getAttribute('fallbackFile', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type,
|
||||
]));
|
||||
$sitesDomain = $platform['sitesDomain'];
|
||||
$domain = ID::unique() . "." . $sitesDomain;
|
||||
|
||||
$site = $site
|
||||
->setAttribute('latestDeploymentId', $deployment->getId())
|
||||
->setAttribute('latestDeploymentInternalId', $deployment->getSequence())
|
||||
->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt())
|
||||
->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', ''));
|
||||
$dbForProject->updateDocument('sites', $site->getId(), new Document([
|
||||
'latestDeploymentId' => $site->getAttribute('latestDeploymentId'),
|
||||
'latestDeploymentInternalId' => $site->getAttribute('latestDeploymentInternalId'),
|
||||
'latestDeploymentCreatedAt' => $site->getAttribute('latestDeploymentCreatedAt'),
|
||||
'latestDeploymentStatus' => $site->getAttribute('latestDeploymentStatus'),
|
||||
]));
|
||||
// TODO: (@Meldiron) Remove after 1.7.x migration
|
||||
$isMd5 = System::getEnv('_APP_RULES_FORMAT') === 'md5';
|
||||
$ruleId = $isMd5 ? md5($domain) : ID::unique();
|
||||
|
||||
$sitesDomain = $platform['sitesDomain'];
|
||||
$domain = ID::unique() . "." . $sitesDomain;
|
||||
$ruleId = md5($domain);
|
||||
$authorization->skip(
|
||||
fn () => $dbForPlatform->createDocument('rules', new Document([
|
||||
'$id' => $ruleId,
|
||||
'projectId' => $project->getId(),
|
||||
'projectInternalId' => $project->getSequence(),
|
||||
'domain' => $domain,
|
||||
'type' => 'deployment',
|
||||
'trigger' => 'deployment',
|
||||
'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(),
|
||||
'deploymentResourceType' => 'site',
|
||||
'deploymentResourceId' => $site->getId(),
|
||||
'deploymentResourceInternalId' => $site->getSequence(),
|
||||
'status' => 'verified',
|
||||
'certificateId' => '',
|
||||
'search' => implode(' ', [$ruleId, $domain]),
|
||||
'owner' => 'Appwrite',
|
||||
'region' => $project->getAttribute('region')
|
||||
]))
|
||||
);
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
$authorization->skip(
|
||||
fn () => $dbForPlatform->createDocument('rules', new Document([
|
||||
'$id' => $ruleId,
|
||||
'projectId' => $project->getId(),
|
||||
'projectInternalId' => $project->getSequence(),
|
||||
'domain' => $domain,
|
||||
'type' => 'deployment',
|
||||
'trigger' => 'deployment',
|
||||
'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(),
|
||||
'deploymentResourceType' => 'site',
|
||||
'deploymentResourceId' => $site->getId(),
|
||||
'deploymentResourceInternalId' => $site->getSequence(),
|
||||
'status' => 'verified',
|
||||
'certificateId' => '',
|
||||
'search' => implode(' ', [$ruleId, $domain]),
|
||||
'owner' => 'Appwrite',
|
||||
'region' => $project->getAttribute('region')
|
||||
]))
|
||||
);
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceSize' => $fileSize,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
|
||||
// Start the build
|
||||
$queueForBuilds
|
||||
->setType(BUILD_TYPE_DEPLOYMENT)
|
||||
->setResource($site)
|
||||
->setDeployment($deployment);
|
||||
} else {
|
||||
if ($deployment->isEmpty()) {
|
||||
$deployment = $dbForProject->createDocument('deployments', new Document([
|
||||
'$id' => $deploymentId,
|
||||
'$permissions' => [
|
||||
Permission::read(Role::any()),
|
||||
Permission::update(Role::any()),
|
||||
Permission::delete(Role::any()),
|
||||
],
|
||||
'resourceInternalId' => $site->getSequence(),
|
||||
'resourceId' => $site->getId(),
|
||||
'resourceType' => 'sites',
|
||||
'buildCommands' => \implode(' && ', $commands),
|
||||
'startCommand' => $site->getAttribute('startCommand', ''),
|
||||
'buildOutput' => $outputDirectory,
|
||||
'adapter' => $site->getAttribute('adapter', ''),
|
||||
'fallbackFile' => $site->getAttribute('fallbackFile', ''),
|
||||
'sourcePath' => $path,
|
||||
'sourceSize' => $fileSize,
|
||||
'totalSize' => $fileSize,
|
||||
'sourceChunksTotal' => $chunks,
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'activate' => $activate,
|
||||
'sourceMetadata' => $metadata,
|
||||
'type' => $type,
|
||||
]));
|
||||
|
||||
$site = $site
|
||||
->setAttribute('latestDeploymentId', $deployment->getId())
|
||||
->setAttribute('latestDeploymentInternalId', $deployment->getSequence())
|
||||
->setAttribute('latestDeploymentCreatedAt', $deployment->getCreatedAt())
|
||||
->setAttribute('latestDeploymentStatus', $deployment->getAttribute('status', ''));
|
||||
$dbForProject->updateDocument('sites', $site->getId(), new Document([
|
||||
'latestDeploymentId' => $site->getAttribute('latestDeploymentId'),
|
||||
'latestDeploymentInternalId' => $site->getAttribute('latestDeploymentInternalId'),
|
||||
'latestDeploymentCreatedAt' => $site->getAttribute('latestDeploymentCreatedAt'),
|
||||
'latestDeploymentStatus' => $site->getAttribute('latestDeploymentStatus'),
|
||||
]));
|
||||
|
||||
$sitesDomain = $platform['sitesDomain'];
|
||||
$domain = ID::unique() . "." . $sitesDomain;
|
||||
$ruleId = md5($domain);
|
||||
$authorization->skip(
|
||||
fn () => $dbForPlatform->createDocument('rules', new Document([
|
||||
'$id' => $ruleId,
|
||||
'projectId' => $project->getId(),
|
||||
'projectInternalId' => $project->getSequence(),
|
||||
'domain' => $domain,
|
||||
'type' => 'deployment',
|
||||
'trigger' => 'deployment',
|
||||
'deploymentId' => $deployment->isEmpty() ? '' : $deployment->getId(),
|
||||
'deploymentInternalId' => $deployment->isEmpty() ? '' : $deployment->getSequence(),
|
||||
'deploymentResourceType' => 'site',
|
||||
'deploymentResourceId' => $site->getId(),
|
||||
'deploymentResourceInternalId' => $site->getSequence(),
|
||||
'status' => 'verified',
|
||||
'certificateId' => '',
|
||||
'search' => implode(' ', [$ruleId, $domain]),
|
||||
'owner' => 'Appwrite',
|
||||
'region' => $project->getAttribute('region')
|
||||
]))
|
||||
);
|
||||
} else {
|
||||
$deployment = $dbForProject->updateDocument('deployments', $deploymentId, new Document([
|
||||
'sourceChunksUploaded' => $chunksUploaded,
|
||||
'sourceMetadata' => $metadata,
|
||||
]));
|
||||
}
|
||||
}
|
||||
|
||||
$metadata = null;
|
||||
|
||||
if ($chunksUploaded === $chunks) {
|
||||
$queueForEvents
|
||||
->setParam('siteId', $site->getId())
|
||||
->setParam('deploymentId', $deployment->getId());
|
||||
}
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
}, timeout: 120.0);
|
||||
} catch (LockContention) {
|
||||
$response->addHeader('Retry-After', '5');
|
||||
throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Deployment upload is busy. Try again.');
|
||||
}
|
||||
|
||||
|
||||
|
||||
$metadata = null;
|
||||
|
||||
$queueForEvents
|
||||
->setParam('siteId', $site->getId())
|
||||
->setParam('deploymentId', $deployment->getId());
|
||||
|
||||
$response
|
||||
->setStatusCode(Response::STATUS_CODE_ACCEPTED)
|
||||
->dynamic($deployment, Response::MODEL_DEPLOYMENT);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1191,6 +1191,144 @@ class FunctionsCustomServerTest extends Scope
|
||||
}, 120000, 500);
|
||||
}
|
||||
|
||||
public function testCreateDeploymentParallelChunksLargeFile(): void
|
||||
{
|
||||
$functionId = $this->setupFunction([
|
||||
'functionId' => ID::unique(),
|
||||
'name' => 'Test Parallel Chunk Deployment',
|
||||
'execute' => [Role::user($this->getUser()['$id'])->toString()],
|
||||
'runtime' => 'node-22',
|
||||
'entrypoint' => 'index.js',
|
||||
'timeout' => 10,
|
||||
]);
|
||||
|
||||
$deploymentId = ID::unique();
|
||||
$tmpDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'appwrite-parallel-function-deployment-' . $deploymentId;
|
||||
|
||||
mkdir($tmpDirectory);
|
||||
|
||||
try {
|
||||
copy(__DIR__ . '/../../../resources/functions/basic/index.js', $tmpDirectory . DIRECTORY_SEPARATOR . 'index.js');
|
||||
file_put_contents($tmpDirectory . DIRECTORY_SEPARATOR . 'large.bin', random_bytes(20 * 1024 * 1024));
|
||||
|
||||
$source = $tmpDirectory . DIRECTORY_SEPARATOR . 'code.tar.gz';
|
||||
Console::execute('cd ' . $tmpDirectory . ' && tar --exclude code.tar.gz -czf code.tar.gz .', '', $this->stdout, $this->stderr);
|
||||
|
||||
$totalSize = filesize($source);
|
||||
$chunkSize = 5 * 1024 * 1024;
|
||||
$chunksTotal = (int) ceil($totalSize / $chunkSize);
|
||||
|
||||
$this->assertGreaterThanOrEqual(4, $chunksTotal, 'Test deployment must span at least 4 chunks');
|
||||
|
||||
$requests = [];
|
||||
$sourceHandle = fopen($source, 'rb');
|
||||
$this->assertNotFalse($sourceHandle, 'Could not open deployment package');
|
||||
|
||||
try {
|
||||
for ($i = 0; $i < $chunksTotal; $i++) {
|
||||
$start = $i * $chunkSize;
|
||||
$end = min($start + $chunkSize, $totalSize) - 1;
|
||||
$length = $end - $start + 1;
|
||||
$chunkPath = $tmpDirectory . DIRECTORY_SEPARATOR . 'chunk-' . $i . '.part';
|
||||
|
||||
fseek($sourceHandle, $start);
|
||||
file_put_contents($chunkPath, fread($sourceHandle, $length));
|
||||
|
||||
$requests[] = [
|
||||
'headers' => [
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
'x-appwrite-id' => $deploymentId,
|
||||
'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize,
|
||||
],
|
||||
'chunkPath' => $chunkPath,
|
||||
];
|
||||
}
|
||||
} finally {
|
||||
fclose($sourceHandle);
|
||||
}
|
||||
|
||||
$responses = [];
|
||||
$endpoint = parse_url($this->client->getEndpoint());
|
||||
$scheme = $endpoint['scheme'] ?? 'http';
|
||||
$host = $endpoint['host'] ?? 'appwrite';
|
||||
$port = $endpoint['port'] ?? ($scheme === 'https' ? 443 : 80);
|
||||
$basePath = rtrim($endpoint['path'] ?? '', '/');
|
||||
|
||||
\Swoole\Coroutine\run(function () use ($basePath, $functionId, $host, $port, $requests, $scheme, &$responses): void {
|
||||
$wg = new \Swoole\Coroutine\WaitGroup();
|
||||
|
||||
foreach ($requests as $index => $request) {
|
||||
$wg->add();
|
||||
\Swoole\Coroutine::create(function () use ($basePath, $functionId, $host, $index, $port, $request, &$responses, $scheme, $wg): void {
|
||||
try {
|
||||
for ($attempt = 0; $attempt < 3; $attempt++) {
|
||||
$client = new \Swoole\Coroutine\Http\Client($host, (int) $port, $scheme === 'https');
|
||||
$client->set([
|
||||
'timeout' => 300,
|
||||
'ssl_verify_peer' => false,
|
||||
'ssl_verify_host' => false,
|
||||
]);
|
||||
$client->setHeaders($request['headers']);
|
||||
$client->setMethod(Client::METHOD_POST);
|
||||
$client->setData([
|
||||
'entrypoint' => 'index.js',
|
||||
'activate' => true,
|
||||
]);
|
||||
$client->addFile($request['chunkPath'], 'code', 'application/x-gzip', 'code.tar.gz');
|
||||
$client->execute($basePath . '/functions/' . $functionId . '/deployments');
|
||||
|
||||
$responses[$index] = [
|
||||
'body' => $client->body,
|
||||
'error' => $client->errMsg,
|
||||
'headers' => $client->headers ?? [],
|
||||
'statusCode' => $client->statusCode,
|
||||
];
|
||||
|
||||
$client->close();
|
||||
|
||||
if ($responses[$index]['statusCode'] !== 429) {
|
||||
break;
|
||||
}
|
||||
|
||||
$retryAfter = (float) ($responses[$index]['headers']['retry-after'] ?? 0.1);
|
||||
\Swoole\Coroutine::sleep(max($retryAfter, 0.1));
|
||||
}
|
||||
} finally {
|
||||
$wg->done();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
$wg->wait();
|
||||
});
|
||||
|
||||
ksort($responses);
|
||||
|
||||
foreach ($responses as $response) {
|
||||
$this->assertSame('', $response['error']);
|
||||
$this->assertContains($response['statusCode'], [202], (string) $response['body']);
|
||||
}
|
||||
|
||||
$this->assertEventually(function () use ($functionId, $deploymentId) {
|
||||
$deployment = $this->getDeployment($functionId, $deploymentId);
|
||||
|
||||
$this->assertEquals(200, $deployment['headers']['status-code']);
|
||||
$this->assertEquals('ready', $deployment['body']['status']);
|
||||
$this->assertEquals($deploymentId, $deployment['body']['$id']);
|
||||
}, 120000, 500);
|
||||
} finally {
|
||||
$this->cleanupFunction($functionId);
|
||||
|
||||
if (is_dir($tmpDirectory)) {
|
||||
foreach (glob($tmpDirectory . DIRECTORY_SEPARATOR . '*') ?: [] as $file) {
|
||||
unlink($file);
|
||||
}
|
||||
rmdir($tmpDirectory);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function testUpdateDeployment(): void
|
||||
{
|
||||
$data = $this->setupTestDeployment();
|
||||
|
||||
@@ -1034,6 +1034,145 @@ class SitesCustomServerTest extends Scope
|
||||
$this->cleanupSite($siteId);
|
||||
}
|
||||
|
||||
public function testCreateDeploymentParallelChunksLargeFile(): void
|
||||
{
|
||||
$siteId = $this->setupSite([
|
||||
'buildRuntime' => 'node-22',
|
||||
'fallbackFile' => '',
|
||||
'framework' => 'other',
|
||||
'name' => 'Test Site Parallel Chunk Deployment',
|
||||
'outputDirectory' => './',
|
||||
'providerBranch' => 'main',
|
||||
'providerRootDirectory' => './',
|
||||
'siteId' => ID::unique()
|
||||
]);
|
||||
|
||||
$deploymentId = ID::unique();
|
||||
$tmpDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'appwrite-parallel-site-deployment-' . $deploymentId;
|
||||
|
||||
mkdir($tmpDirectory);
|
||||
|
||||
try {
|
||||
file_put_contents($tmpDirectory . DIRECTORY_SEPARATOR . 'index.html', '<html><body>Hello World</body></html>');
|
||||
file_put_contents($tmpDirectory . DIRECTORY_SEPARATOR . 'large.bin', random_bytes(20 * 1024 * 1024));
|
||||
|
||||
$source = $tmpDirectory . DIRECTORY_SEPARATOR . 'code.tar.gz';
|
||||
Console::execute('cd ' . $tmpDirectory . ' && tar --exclude code.tar.gz -czf code.tar.gz .', '', $this->stdout, $this->stderr);
|
||||
|
||||
$totalSize = filesize($source);
|
||||
$chunkSize = 5 * 1024 * 1024;
|
||||
$chunksTotal = (int) ceil($totalSize / $chunkSize);
|
||||
|
||||
$this->assertGreaterThanOrEqual(4, $chunksTotal, 'Test deployment must span at least 4 chunks');
|
||||
|
||||
$requests = [];
|
||||
$sourceHandle = fopen($source, 'rb');
|
||||
$this->assertNotFalse($sourceHandle, 'Could not open deployment package');
|
||||
|
||||
try {
|
||||
for ($i = 0; $i < $chunksTotal; $i++) {
|
||||
$start = $i * $chunkSize;
|
||||
$end = min($start + $chunkSize, $totalSize) - 1;
|
||||
$length = $end - $start + 1;
|
||||
$chunkPath = $tmpDirectory . DIRECTORY_SEPARATOR . 'chunk-' . $i . '.part';
|
||||
|
||||
fseek($sourceHandle, $start);
|
||||
file_put_contents($chunkPath, fread($sourceHandle, $length));
|
||||
|
||||
$requests[] = [
|
||||
'headers' => [
|
||||
'x-appwrite-project' => $this->getProject()['$id'],
|
||||
'x-appwrite-key' => $this->getProject()['apiKey'],
|
||||
'x-appwrite-id' => $deploymentId,
|
||||
'content-range' => 'bytes ' . $start . '-' . $end . '/' . $totalSize,
|
||||
],
|
||||
'chunkPath' => $chunkPath,
|
||||
];
|
||||
}
|
||||
} finally {
|
||||
fclose($sourceHandle);
|
||||
}
|
||||
|
||||
$responses = [];
|
||||
$endpoint = parse_url($this->client->getEndpoint());
|
||||
$scheme = $endpoint['scheme'] ?? 'http';
|
||||
$host = $endpoint['host'] ?? 'appwrite';
|
||||
$port = $endpoint['port'] ?? ($scheme === 'https' ? 443 : 80);
|
||||
$basePath = rtrim($endpoint['path'] ?? '', '/');
|
||||
|
||||
\Swoole\Coroutine\run(function () use ($basePath, $host, $port, $requests, $scheme, $siteId, &$responses): void {
|
||||
$wg = new \Swoole\Coroutine\WaitGroup();
|
||||
|
||||
foreach ($requests as $index => $request) {
|
||||
$wg->add();
|
||||
\Swoole\Coroutine::create(function () use ($basePath, $host, $index, $port, $request, &$responses, $scheme, $siteId, $wg): void {
|
||||
try {
|
||||
for ($attempt = 0; $attempt < 3; $attempt++) {
|
||||
$client = new \Swoole\Coroutine\Http\Client($host, (int) $port, $scheme === 'https');
|
||||
$client->set([
|
||||
'timeout' => 300,
|
||||
'ssl_verify_peer' => false,
|
||||
'ssl_verify_host' => false,
|
||||
]);
|
||||
$client->setHeaders($request['headers']);
|
||||
$client->setMethod(Client::METHOD_POST);
|
||||
$client->setData([
|
||||
'activate' => true,
|
||||
]);
|
||||
$client->addFile($request['chunkPath'], 'code', 'application/x-gzip', 'code.tar.gz');
|
||||
$client->execute($basePath . '/sites/' . $siteId . '/deployments');
|
||||
|
||||
$responses[$index] = [
|
||||
'body' => $client->body,
|
||||
'error' => $client->errMsg,
|
||||
'headers' => $client->headers ?? [],
|
||||
'statusCode' => $client->statusCode,
|
||||
];
|
||||
|
||||
$client->close();
|
||||
|
||||
if ($responses[$index]['statusCode'] !== 429) {
|
||||
break;
|
||||
}
|
||||
|
||||
$retryAfter = (float) ($responses[$index]['headers']['retry-after'] ?? 0.1);
|
||||
\Swoole\Coroutine::sleep(max($retryAfter, 0.1));
|
||||
}
|
||||
} finally {
|
||||
$wg->done();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
$wg->wait();
|
||||
});
|
||||
|
||||
ksort($responses);
|
||||
|
||||
foreach ($responses as $response) {
|
||||
$this->assertSame('', $response['error']);
|
||||
$this->assertContains($response['statusCode'], [202], (string) $response['body']);
|
||||
}
|
||||
|
||||
$this->assertEventually(function () use ($siteId, $deploymentId) {
|
||||
$deployment = $this->getDeployment($siteId, $deploymentId);
|
||||
|
||||
$this->assertEquals(200, $deployment['headers']['status-code']);
|
||||
$this->assertEquals('ready', $deployment['body']['status']);
|
||||
$this->assertEquals($deploymentId, $deployment['body']['$id']);
|
||||
}, 120000, 500);
|
||||
} finally {
|
||||
$this->cleanupSite($siteId);
|
||||
|
||||
if (is_dir($tmpDirectory)) {
|
||||
foreach (glob($tmpDirectory . DIRECTORY_SEPARATOR . '*') ?: [] as $file) {
|
||||
unlink($file);
|
||||
}
|
||||
rmdir($tmpDirectory);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function testCreateDeployment()
|
||||
{
|
||||
$siteId = $this->setupSite([
|
||||
|
||||
@@ -1376,7 +1376,7 @@ trait StorageBase
|
||||
|
||||
public function testCreateBucketFileParallelChunksLargeFile(): void
|
||||
{
|
||||
$totalSize = (int) ($_ENV['APPWRITE_TEST_PARALLEL_UPLOAD_SIZE'] ?? 20 * 1024 * 1024);
|
||||
$totalSize = 20 * 1024 * 1024;
|
||||
$chunkSize = 5 * 1024 * 1024;
|
||||
$chunksTotal = (int) ceil($totalSize / $chunkSize);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user