fix(data-transfer): fix large transfer crashes; show transfer progress (#23479)

* enhancement: update progress per chunk

* fix: check stageprogress exists to make ts happy

* chore: split progress tracker into two methods

* test: fix lint

* enhancement: display readable times

* fix: speed indication for assets

* fix: restore speed indicator and fix double counting

* chore: clean up code

* fix: data transfer memory leak

* fix: yarn.lock

* chore: fix memory logging

* ci: complex project remote transfer

* enhancement: optimized transfer mode

* test(cli): data transfer and env passthrough

* chore: only send one message warning of legacy mode

* enhancement: show transfer totals and estimated time

* test(cli): fix deterministic transfer test files

* fix: push and pull shared utils backwards compatibility

* fix(data-transfer): extract legacy asset chunk parsing and tighten transfer logging/test coverage

* enhancement: checksum negotiation

* enhancement: show skipped file warnings on client

* fix: transfer diagnostics

* test: fix open handle

* fix: clear stall timeout for assets

* chore: fix misleading comments and variables

* test: fix misleading test

* test: fix typo

* test: make checks deterministic, less flaky

* enhancement(data-transfer): speed up asset totals; widen assets start reply window on remote pull

* fix(data-transfer): harden WebSocket JSON serialization for transfer frames

* fix(data-transfer): more transfer hardening

* test: fix test imports

* fix: await async write

* fix(data-transfer): resolve push transfer deadlock and harden async writes

- Extract createAssetsDestinationWritable so Writable callbacks run before
  uploadStream completes (same WS batch as PassThrough chunks).
- Add writable-async-write (write callback + drain/finished race; avoid hang
  on destroy).
- Wire push/pull, remote-source, file & directory sources to shared write().
- Fire-and-forget pull flush: Promise.resolve(flush).catch(onError); guard
  missing stream inside try.
- Add regression tests (assets writable, writable-async-write, handler checks).

* fix(data-transfer): write push stream batches sequentially

Use a for-loop with await write() instead of Promise.all over msg.data
so non-asset stages respect one in-flight write per objectMode Writable
and backpressure from writable-async-write.

- Validate minChunksForBackpressure in assertReadStreamBackpressure
- Add engine test for non-Buffer asset chunk byte progress (counts as 1)
- Assert push.ts keeps sequential msg.data handling in static handler test

* fix(data-transfer): align push streamAsset with remote-source and harden tests

- Push handler: combine stream/end under one branch, error when start is missing
  or action is invalid; shorten stage write comments.
- Engine version-matching tests: use a fresh createDestination() per engine so
  parallel transfers do not share destination writables (MaxListeners warnings).
- File destination tests: mock createWriteStream with a new Writable per call.
- CLI transfer tests: mock progress.stream so transfer::finish runs after transfer
  and clears the progress setInterval (fixes Jest worker hang).
- Misc test cleanup: assets-destination timeout clearTimeout, collect listeners,
  writable-async-write teardown; tighten push/static test descriptions.

* test: remove parity test

* fix(data-transfer): harden collect() and stabilize transfer tests

- collect(): settle once, remove listeners on resolve/reject, avoid double completion
- engine tests: add expectHeapGrowthWithinNoise for heap smoke checks
- CLI transfer tests: console spies in beforeAll; jest.restoreAllMocks in afterAll
- stream test: remove removeAllListeners workaround
This commit is contained in:
Ben Irvin
2026-04-20 16:08:05 +02:00
committed by GitHub
parent 43c03c077f
commit bac520e9f8
71 changed files with 5273 additions and 482 deletions
+5
View File
@@ -4,6 +4,11 @@ const serverConfig = ({ env }) => ({
app: {
keys: env.array('APP_KEYS', ['toBeModified1', 'toBeModified2']),
},
transfer: {
remote: {
enabled: true,
},
},
});
export default serverConfig;
+1 -1
View File
@@ -42,7 +42,7 @@
"typescript": "^5"
},
"engines": {
"node": ">=18.0.0 <=22.x.x",
"node": ">=20.0.0 <=24.x.x",
"npm": ">=6.0.0"
},
"strapi": {
+1 -1
View File
@@ -30,7 +30,7 @@
"typescript": "^5"
},
"engines": {
"node": ">=18.0.0 <=22.x.x",
"node": ">=20.0.0 <=24.x.x",
"npm": ">=6.0.0"
},
"strapi": {
@@ -1,7 +1,84 @@
import { Readable } from 'stream';
import { Readable, Writable } from 'stream';
import { pipeline } from 'stream/promises';
import type { Core } from '@strapi/types';
import type { ITransferEngine, ISourceProvider, IDestinationProvider } from '../../types';
/**
* Create a slow Writable that applies backpressure (low highWaterMark, delayed write).
* Used to verify that source streams pause when the consumer is slow.
*/
export const createSlowWritable = <T = unknown>(
options: {
objectMode?: boolean;
highWaterMark?: number;
delayMs?: number;
onChunk?: (chunk: T) => void;
} = {}
): { writable: Writable; chunks: T[] } => {
const { objectMode = true, highWaterMark = 1, delayMs = 10, onChunk } = options;
const chunks: T[] = [];
const writable = new Writable({
objectMode,
highWaterMark,
write(chunk: T, _encoding, callback) {
chunks.push(chunk);
onChunk?.(chunk);
setTimeout(callback, delayMs);
},
});
return { writable, chunks };
};
/**
* Run a backpressure test on a Readable: pipe to a slow consumer and assert the source stream
* was paused at least once (proving backpressure is applied). Returns collected chunks for integrity checks.
*/
export const assertReadStreamBackpressure = async <T = unknown>(
stream: Readable,
options: { delayMs?: number; minChunksForBackpressure?: number } = {}
): Promise<{ sourcePaused: boolean; chunks: T[] }> => {
const { delayMs = 10, minChunksForBackpressure } = options;
let sourcePaused = false;
const originalPause = stream.pause.bind(stream);
stream.pause = function (this: Readable) {
sourcePaused = true;
return originalPause();
};
const { writable, chunks } = createSlowWritable<T>({ delayMs, highWaterMark: 1 });
await pipeline(stream, writable);
if (minChunksForBackpressure !== undefined && chunks.length < minChunksForBackpressure) {
throw new Error(
`assertReadStreamBackpressure: need at least ${minChunksForBackpressure} chunk(s) to meaningfully test backpressure, got ${chunks.length}`
);
}
return {
sourcePaused,
chunks,
};
};
/**
* Run a backpressure test on a Writable: pipe a fast Readable (many chunks) into it and assert
* the readable was paused (proving the destination applies backpressure).
*/
export const assertWriteStreamBackpressure = async <T = unknown>(
writable: Writable,
chunks: T[]
): Promise<{ sourcePaused: boolean }> => {
const source = Readable.from(chunks, { objectMode: true });
let sourcePaused = false;
const originalPause = source.pause.bind(source);
source.pause = function (this: Readable) {
sourcePaused = true;
return originalPause();
};
await pipeline(source, writable);
return { sourcePaused };
};
/**
* Collect every entity in a Readable stream
*/
@@ -4,6 +4,7 @@ import path from 'path';
import { Readable } from 'stream';
import { createLocalDirectorySourceProvider } from '..';
import { assertReadStreamBackpressure } from '../../../../__tests__/test-utils';
const minimalMetadata = { strapi: { version: '5.0.0' }, createdAt: new Date().toISOString() };
@@ -58,6 +59,28 @@ describe('Directory source provider', () => {
stream.destroy();
});
test('entities read stream pauses under backpressure (slow consumer)', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'dts-dir-bp-'));
await fs.writeJson(path.join(dir, 'metadata.json'), minimalMetadata);
await fs.ensureDir(path.join(dir, 'entities'));
const entityLines = Array.from(
{ length: 25 },
(_, i) => `${JSON.stringify({ type: 'api::test.test', id: i + 1, data: {} })}\n`
).join('');
await fs.writeFile(path.join(dir, 'entities', 'entities_00000.jsonl'), entityLines);
const provider = createLocalDirectorySourceProvider({ directory: { path: dir } });
await provider.bootstrap({ report: jest.fn() } as never);
const stream = provider.createEntitiesReadStream();
const { sourcePaused, chunks } = await assertReadStreamBackpressure(stream, {
delayMs: 12,
});
expect(sourcePaused).toBe(true);
expect(chunks).toHaveLength(25);
}, 8000);
test('streams entities from jsonl shards in order', async () => {
const dir = await fs.mkdtemp(path.join(tmpdir(), 'dts-dir-'));
await fs.writeJson(path.join(dir, 'metadata.json'), minimalMetadata);
@@ -12,6 +12,7 @@ import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
import * as utils from '../../../utils';
import { write } from '../../../utils/writable-async-write';
import { ProviderInitializationError, ProviderTransferError } from '../../../errors/providers';
import { unknownPathToPosix } from '../../../file/providers/source/utils';
@@ -196,7 +197,7 @@ class LocalDirectorySourceProvider implements ISourceProvider {
stats: { size: stat.size },
stream: fs.createReadStream(absUpload),
};
outStream.write(asset);
await write(outStream, asset);
}
}
outStream.end();
@@ -246,7 +247,7 @@ class LocalDirectorySourceProvider implements ISourceProvider {
try {
for await (const chunk of stream) {
outStream.write(chunk);
await write(outStream, chunk);
}
} catch (e: unknown) {
outStream.destroy(
@@ -1,6 +1,9 @@
import { posix, win32 } from 'path';
import path, { posix, win32 } from 'path';
import os from 'os';
import fs from 'fs-extra';
import { cloneDeep, get, set } from 'lodash/fp';
import { Readable, Writable } from 'stream-chain';
import { pipeline } from 'stream/promises';
import type { Struct } from '@strapi/types';
import { createTransferEngine, TRANSFER_STAGES } from '..';
@@ -14,13 +17,19 @@ import type {
ITransferEngineOptions,
TransferFilterPreset,
} from '../../../types';
import {
extendExpectForDataTransferTests,
providerStages,
sourceStages,
} from '../../__tests__/test-utils';
import { extendExpectForDataTransferTests } from '../../__tests__/test-utils';
import { TransferEngineValidationError } from '../errors';
/**
* Parallel Jest + V8 can move `heapUsed` by tens of MB unrelated to transfer payload.
* Use this as a loose smoke bound; structural tests (order, file bytes) are the real checks.
*/
function expectHeapGrowthWithinNoise(heapGrowth: number, totalBytes: number) {
const limit = Math.max(totalBytes * 150, 32 * 1024 * 1024);
expect(heapGrowth).toBeLessThan(limit);
}
const getMockSourceStream = (data: Iterable<unknown>) => Readable.from(data);
const defaultLinksData: Array<ILink> = [
@@ -690,20 +699,23 @@ describe('Transfer engine', () => {
const source = createSource();
const engine = createTransferEngine(source, completeDestination, defaultOptions);
let calls = 0;
const progressEvents: Record<string, number> = {};
engine.progress.stream.on('stage::progress', ({ stage, data }) => {
expect(TRANSFER_STAGES.includes(stage)).toBe(true);
expect(data).toMatchObject(engine.progress.data);
calls += 1;
progressEvents[stage] = (progressEvents[stage] || 0) + 1;
});
await engine.transfer();
// Two values are emitted by default for each stage
// TODO: this is no longer true, we should be checking the sum of the various mocked streams
const itemPerStage = 3;
// Each stage should emit at least one progress event
TRANSFER_STAGES.forEach((stage) => {
expect(progressEvents[stage]).toBeGreaterThanOrEqual(1);
});
expect(calls).toEqual((sourceStages.length - providerStages.length) * itemPerStage);
// For assets, the number of progress events should match the number of chunks plus one for each asset's 'end' event
// (from getAssetsMockSourceStream default: [ [1,2,3], [4,5,6,7,8,9] ] => 3 + 6 = 9, plus 2 'end' events = 11)
expect(progressEvents.assets).toBe(11);
});
test("emits 'stage::start' events", async () => {
@@ -722,6 +734,31 @@ describe('Transfer engine', () => {
expect(calls).toEqual(TRANSFER_STAGES.length);
});
test('merges source getStageTotals into assets progress before stage::start', async () => {
const source = createSource();
source.getStageTotals = jest.fn().mockResolvedValue({ totalBytes: 12_345, totalCount: 7 });
const engine = createTransferEngine(source, completeDestination, defaultOptions);
let assetsAtStart: Record<string, unknown> | undefined;
engine.progress.stream.on('stage::start', ({ stage, data }) => {
if (stage === 'assets' && data.assets) {
// Snapshot: `data` is the live progress object and mutates during the stage.
assetsAtStart = { ...data.assets };
}
});
await engine.transfer();
expect(source.getStageTotals).toHaveBeenCalledWith('assets');
expect(assetsAtStart).toMatchObject({
totalBytes: 12_345,
totalCount: 7,
count: 0,
bytes: 0,
});
});
test("emits 'stage::finish' events", async () => {
const source = createSource();
const engine = createTransferEngine(source, completeDestination, defaultOptions);
@@ -902,7 +939,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
expect(
(async () => {
await engine.transfer();
@@ -924,7 +961,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
expect(
(async () => {
await engine.transfer();
@@ -937,7 +974,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
expect(
(async () => {
await engine.transfer();
@@ -960,7 +997,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
await expect(
(async () => {
await engine.transfer();
@@ -975,7 +1012,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
await expect(
(async () => {
await engine.transfer();
@@ -999,7 +1036,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
await expect(
(async () => {
await engine.transfer();
@@ -1014,7 +1051,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
await expect(
(async () => {
await engine.transfer();
@@ -1038,7 +1075,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
await expect(
(async () => {
await engine.transfer();
@@ -1053,7 +1090,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
await expect(
(async () => {
await engine.transfer();
@@ -1076,7 +1113,7 @@ describe('Transfer engine', () => {
modifiedMetadata.strapi.version = version;
const source = createSource();
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
const engine = createTransferEngine(source, completeDestination, options);
const engine = createTransferEngine(source, createDestination(), options);
await expect(
(async () => {
await engine.transfer();
@@ -1120,7 +1157,7 @@ describe('Transfer engine', () => {
createEntitiesReadStream() {
const stream = Readable.from(sourceData, { objectMode: true });
const originalPause = stream.pause.bind(stream);
stream.pause = function () {
stream.pause = function pause() {
sourcePaused = true;
return originalPause();
};
@@ -1202,7 +1239,7 @@ describe('Transfer engine', () => {
createAssetsReadStream() {
const stream = getAssetsMockSourceStream(assetData);
const originalPause = stream.pause.bind(stream);
stream.pause = function () {
stream.pause = function pause() {
sourcePaused = true;
return originalPause();
};
@@ -1226,4 +1263,512 @@ describe('Transfer engine', () => {
);
}, 3000);
});
describe('asset stream data and memory', () => {
/**
* Proves that asset stream bytes reach the destination. If the progress tracker
* consumed the stream (old bug), the destination would receive 0 bytes per asset.
*/
test('destination receives full stream bytes for each asset (no double-consumption)', async () => {
const expectedBytesPerAsset = [100, 250, 75];
const assetData: IAsset[] = expectedBytesPerAsset.map((size, i) => ({
filename: `file${i}.jpg`,
filepath: posix.join(__dirname, `file${i}.jpg`),
stats: { size },
stream: Readable.from(Array.from({ length: size }, () => Buffer.alloc(1))),
metadata: {
hash: `hash${i}`,
ext: '.jpg',
id: i,
name: '',
mime: 'image/jpeg',
size: 0,
url: '',
},
}));
const bytesReceivedPerAsset: number[] = [];
const destination: IDestinationProvider = {
...completeDestination,
createAssetsWriteStream() {
return new Writable({
objectMode: true,
async write(asset: IAsset, _encoding, callback) {
if (!asset?.stream || typeof asset.stream.pipe !== 'function') {
bytesReceivedPerAsset.push(0);
return callback();
}
let bytes = 0;
const counter = new Writable({
write(chunk: Buffer | unknown, _enc, cb) {
bytes += Buffer.isBuffer(chunk) ? chunk.length : 1;
cb();
},
});
try {
await pipeline(asset.stream, counter);
} catch (e) {
return callback(e instanceof Error ? e : new Error(String(e)));
}
bytesReceivedPerAsset.push(bytes);
callback();
},
});
},
};
const source = createSource({ assets: assetData });
const engine = createTransferEngine(source, destination, defaultOptions);
await engine.transfer();
expect(bytesReceivedPerAsset).toEqual(expectedBytesPerAsset);
}, 5000);
/**
* Documents {@link TransferEngine}s per-chunk progress rule: only `Buffer` chunks use
* `.length`; anything else counts as 1 byte (cosmetic for ETA when chunk shapes are odd).
*/
test('progress byte totals: a non-Buffer chunk contributes 1 to the byte counter', async () => {
const assetData: IAsset[] = [
{
filename: 'odd-chunk.bin',
filepath: posix.join(__dirname, 'odd-chunk.bin'),
stats: { size: 3 },
stream: Readable.from([{ not: 'a buffer' }, Buffer.alloc(2)] as unknown[]),
metadata: {
hash: 'h2',
ext: '.bin',
id: 0,
name: '',
mime: 'application/octet-stream',
size: 0,
url: '',
},
},
];
const source = createSource({ assets: assetData });
const engine = createTransferEngine(source, completeDestination, defaultOptions);
await engine.transfer();
expect(engine.progress.data.assets?.bytes).toBe(3);
}, 5000);
/**
* Smoke check that we are not holding the whole payload many times over in heap.
* (See {@link expectHeapGrowthWithinNoise} — absolute noise dominates small transfers.)
*/
test('heap growth during asset transfer stays bounded (streaming, not buffering)', async () => {
const assetCount = 15;
const bytesPerAsset = 50 * 1024; // 50KB each
const totalBytes = assetCount * bytesPerAsset;
const assetData: IAsset[] = Array.from({ length: assetCount }, (_, i) => ({
filename: `large${i}.bin`,
filepath: posix.join(__dirname, `large${i}.bin`),
stats: { size: bytesPerAsset },
stream: Readable.from(
Array.from({ length: bytesPerAsset / 1024 }, () => Buffer.alloc(1024))
),
metadata: {
hash: `h${i}`,
ext: '.bin',
id: i,
name: '',
mime: 'application/octet-stream',
size: 0,
url: '',
},
}));
const initialHeap = process.memoryUsage().heapUsed;
const source = createSource({ assets: assetData });
const engine = createTransferEngine(source, completeDestination, defaultOptions);
await engine.transfer();
const finalHeap = process.memoryUsage().heapUsed;
const heapGrowth = finalHeap - initialHeap;
expectHeapGrowthWithinNoise(heapGrowth, totalBytes);
}, 10000);
});
describe('asset transfer integration (order, content, memory)', () => {
/**
* Full transfer path: source → engine (progress tracker) → destination that
* "writes" each asset to a temp file. Verifies:
* 1) Assets are received and written in correct order (no race).
* 2) Each file's content matches the source (no mixing/corruption).
* 3) Heap during transfer stays within a loose smoke bound (see {@link expectHeapGrowthWithinNoise}).
*/
test('full asset transfer: order preserved, content correct, memory bounded', async () => {
const assetCount = 6;
const bytesPerAsset = 10 * 1024; // 10KB each, unique byte value per asset
const tmpDir = path.join(os.tmpdir(), `strapi-dt-integration-${Date.now()}`);
await fs.ensureDir(tmpDir);
const assetData: IAsset[] = Array.from({ length: assetCount }, (_, i) => ({
filename: `media-${i}.bin`,
filepath: posix.join(__dirname, `media-${i}.bin`),
stats: { size: bytesPerAsset },
stream: Readable.from([Buffer.alloc(bytesPerAsset, i)]),
metadata: {
hash: `hash${i}`,
ext: '.bin',
id: i,
name: `media-${i}`,
mime: 'application/octet-stream',
size: bytesPerAsset,
url: '',
},
}));
const writeOrder: number[] = [];
const memorySamples: number[] = [];
let memoryInterval: ReturnType<typeof setInterval> | null = null;
const destination: IDestinationProvider = {
...createDestination(),
createAssetsWriteStream: jest.fn().mockResolvedValue(
new Writable({
objectMode: true,
async write(asset: IAsset, _encoding, callback) {
const index = assetData.findIndex((a) => a.filename === asset.filename);
writeOrder.push(index);
const outPath = path.join(tmpDir, asset.filename);
if (!asset?.stream || typeof asset.stream.pipe !== 'function') {
return callback(new Error('Missing or invalid asset stream'));
}
try {
await pipeline(asset.stream, fs.createWriteStream(outPath));
} catch (e) {
return callback(e instanceof Error ? e : new Error(String(e)));
}
callback();
},
})
),
};
const source = createSource({
assets: assetData,
schemas: Object.values(schemas) as Struct.Schema[],
entities: [],
links: [],
configuration: [],
});
memorySamples.push(process.memoryUsage().heapUsed);
memoryInterval = setInterval(() => {
memorySamples.push(process.memoryUsage().heapUsed);
}, 25);
const engine = createTransferEngine(source, destination, {
...defaultOptions,
only: ['files'],
});
await engine.transfer();
if (memoryInterval) {
clearInterval(memoryInterval);
}
memorySamples.push(process.memoryUsage().heapUsed);
const initialHeap = memorySamples[0];
const maxHeap = Math.max(...memorySamples);
const heapGrowth = maxHeap - initialHeap;
const totalBytes = assetCount * bytesPerAsset;
expectHeapGrowthWithinNoise(heapGrowth, totalBytes);
expect(writeOrder).toHaveLength(assetCount);
expect(writeOrder).toEqual([0, 1, 2, 3, 4, 5]);
for (let i = 0; i < assetCount; i += 1) {
const content = await fs.readFile(path.join(tmpDir, `media-${i}.bin`));
expect(content.length).toBe(bytesPerAsset);
expect(content.every((b) => b === i)).toBe(true);
}
await fs.remove(tmpDir).catch(() => {});
}, 15000);
});
describe('backpressure (schemas, links, configuration)', () => {
test('schemas source stream pauses under backpressure and data integrity is maintained', async () => {
const schemaData = getSchemasMockSourceStream();
const schemaChunks = await new Promise<Struct.Schema[]>((resolve, reject) => {
const chunks: Struct.Schema[] = [];
schemaData.on('data', (chunk: Struct.Schema) => chunks.push(chunk));
schemaData.on('end', () => resolve(chunks));
schemaData.on('error', reject);
});
let sourcePaused = false;
const processedData: Struct.Schema[] = [];
const slowDestination: IDestinationProvider = {
...completeDestination,
createSchemasWriteStream() {
return new Writable({
objectMode: true,
highWaterMark: 1,
write(chunk, _encoding, callback) {
processedData.push(chunk);
setTimeout(callback, 10);
},
});
},
};
const source = {
...createSource({ schemas: schemaChunks }),
createSchemasReadStream() {
const stream = getSchemasMockSourceStream(schemaChunks);
const originalPause = stream.pause.bind(stream);
stream.pause = function () {
sourcePaused = true;
return originalPause();
};
return stream;
},
};
const engine = createTransferEngine(source, slowDestination, defaultOptions);
await engine.transfer();
expect(sourcePaused).toBe(true);
expect(processedData).toEqual(schemaChunks);
}, 2000);
test('links source stream pauses under backpressure and data integrity is maintained', async () => {
const linksData = [...defaultLinksData];
let sourcePaused = false;
const processedData: ILink[] = [];
const slowDestination: IDestinationProvider = {
...completeDestination,
createLinksWriteStream() {
return new Writable({
objectMode: true,
highWaterMark: 1,
write(chunk, _encoding, callback) {
processedData.push(chunk);
setTimeout(callback, 10);
},
});
},
};
const source = {
...createSource({ links: linksData }),
createLinksReadStream() {
const stream = getLinksMockSourceStream(linksData);
const originalPause = stream.pause.bind(stream);
stream.pause = function () {
sourcePaused = true;
return originalPause();
};
return stream;
},
};
const engine = createTransferEngine(source, slowDestination, defaultOptions);
await engine.transfer();
expect(sourcePaused).toBe(true);
expect(processedData).toHaveLength(linksData.length);
expect(processedData).toEqual(expect.arrayContaining(linksData));
}, 2000);
test('configuration source stream pauses under backpressure and data integrity is maintained', async () => {
const configData: IConfiguration[] = [
{ type: 'core-store', value: { key: 'foo', value: 'alice' } },
{ type: 'core-store', value: { key: 'bar', value: 'bob' } },
{ type: 'core-store', value: { key: 'baz', value: 'charlie' } },
];
let sourcePaused = false;
const processedData: IConfiguration[] = [];
const slowDestination: IDestinationProvider = {
...completeDestination,
createConfigurationWriteStream() {
return new Writable({
objectMode: true,
highWaterMark: 1,
write(chunk, _encoding, callback) {
processedData.push(chunk);
setTimeout(callback, 10);
},
});
},
};
const source = {
...createSource({ configuration: configData }),
createConfigurationReadStream() {
const stream = getConfigurationMockSourceStream(configData);
const originalPause = stream.pause.bind(stream);
stream.pause = function () {
sourcePaused = true;
return originalPause();
};
return stream;
},
};
const engine = createTransferEngine(source, slowDestination, defaultOptions);
await engine.transfer();
expect(sourcePaused).toBe(true);
expect(processedData).toEqual(configData);
}, 2000);
});
describe('stream cleanup and memory', () => {
test('all stage streams are destroyed after successful transfer', async () => {
const createdReadStreams: Readable[] = [];
const createdWriteStreams: Writable[] = [];
const source = {
...createSource(),
createEntitiesReadStream: jest.fn().mockImplementation(async () => {
const s = getEntitiesMockSourceStream();
createdReadStreams.push(s);
return s;
}),
createLinksReadStream: jest.fn().mockImplementation(() => {
const s = getLinksMockSourceStream();
createdReadStreams.push(s);
return s;
}),
createAssetsReadStream: jest.fn().mockImplementation(async () => {
const s = getAssetsMockSourceStream();
createdReadStreams.push(s);
return s;
}),
createConfigurationReadStream: jest.fn().mockImplementation(() => {
const s = getConfigurationMockSourceStream();
createdReadStreams.push(s);
return s;
}),
createSchemasReadStream: jest.fn().mockImplementation(() => {
const s = getSchemasMockSourceStream();
createdReadStreams.push(s);
return s;
}),
};
const destination = {
...createDestination(),
createEntitiesWriteStream: jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
createdWriteStreams.push(w);
return w;
}),
createLinksWriteStream: jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
createdWriteStreams.push(w);
return w;
}),
createAssetsWriteStream: jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
createdWriteStreams.push(w);
return w;
}),
createConfigurationWriteStream: jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
createdWriteStreams.push(w);
return w;
}),
createSchemasWriteStream: jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
createdWriteStreams.push(w);
return w;
}),
};
const engine = createTransferEngine(source, destination, defaultOptions);
await engine.transfer();
createdReadStreams.forEach((stream) => {
expect(stream.destroyed).toBe(true);
});
createdWriteStreams.forEach((stream) => {
expect(stream.destroyed).toBe(true);
});
});
test('repeated transfers do not leave streams undestroyed', async () => {
const transferCount = 5;
const allReadStreams: Readable[] = [];
const allWriteStreams: Writable[] = [];
const source = createSource();
source.createEntitiesReadStream = jest.fn().mockImplementation(async () => {
const s = getEntitiesMockSourceStream();
allReadStreams.push(s);
return s;
});
source.createLinksReadStream = jest.fn().mockImplementation(() => {
const s = getLinksMockSourceStream();
allReadStreams.push(s);
return s;
});
source.createSchemasReadStream = jest.fn().mockImplementation(() => {
const s = getSchemasMockSourceStream();
allReadStreams.push(s);
return s;
});
source.createAssetsReadStream = jest.fn().mockImplementation(async () => {
const s = getAssetsMockSourceStream();
allReadStreams.push(s);
return s;
});
source.createConfigurationReadStream = jest.fn().mockImplementation(() => {
const s = getConfigurationMockSourceStream();
allReadStreams.push(s);
return s;
});
const destination = createDestination();
destination.createEntitiesWriteStream = jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
allWriteStreams.push(w);
return w;
});
destination.createLinksWriteStream = jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
allWriteStreams.push(w);
return w;
});
destination.createSchemasWriteStream = jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
allWriteStreams.push(w);
return w;
});
destination.createAssetsWriteStream = jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
allWriteStreams.push(w);
return w;
});
destination.createConfigurationWriteStream = jest.fn().mockImplementation(async () => {
const w = getMockDestinationStream();
allWriteStreams.push(w);
return w;
});
const engine = createTransferEngine(source, destination, defaultOptions);
for (let i = 0; i < transferCount; i += 1) {
await engine.transfer();
}
allReadStreams.forEach((stream) => {
expect(stream.destroyed).toBe(true);
});
allWriteStreams.forEach((stream) => {
expect(stream.destroyed).toBe(true);
});
});
});
});
+116 -4
View File
@@ -32,6 +32,7 @@ import type {
ErrorHandlerContext,
ErrorHandlers,
ErrorCode,
StageProgress,
} from '../../types';
import type { Diff } from '../utils/json';
@@ -307,8 +308,7 @@ class TransferEngine<
}
/**
* Create and return a PassThrough stream.
*
* Create and return a PassThrough stream for per-object progress tracking.
* Upon writing data into it, it'll update the Engine's transfer progress data and trigger stage update events.
*/
#progressTracker(
@@ -328,6 +328,90 @@ class TransferEngine<
});
}
/**
* Create and return a PassThrough stream for per-chunk progress tracking (used for assets).
* Pipes each asset's binary stream through a Transform that counts bytes and forwards chunks,
* then replaces asset.stream with that transform so the destination has a single consumer.
* This avoids consuming the stream (which would leave the destination with an empty stream)
* and ensures backpressure is applied so memory is not held for the entire transfer.
*/
#progressTrackerChunks(
stage: TransferStage,
aggregate?: {
key?(value: unknown): string;
}
) {
const updateAggregateBytes = this.#updateAggregateBytes.bind(this);
const incrementAggregateCount = this.#incrementAggregateCount.bind(this);
const emitStageUpdate = this.#emitStageUpdate.bind(this);
return new PassThrough({
objectMode: true,
transform: (asset, _encoding, callback) => {
if (!asset?.stream || typeof asset.stream.pipe !== 'function') {
return callback(null, asset);
}
const key = aggregate?.key?.(asset);
if (!this.progress.data[stage]) {
this.progress.data[stage] = { count: 0, bytes: 0, startTime: Date.now() };
}
const stageProgress = this.progress.data[stage];
if (!stageProgress) {
throw new TransferEngineError('fatal', 'Stage progress data not found');
}
const progressTransform = new Transform({
objectMode: true,
transform(chunk: Buffer | unknown, _enc, cb) {
// Asset file reads should yield Buffers; avoid skewing totals if not.
const byteLength = Buffer.isBuffer(chunk) ? chunk.length : 1;
stageProgress.bytes += byteLength;
if (key) {
updateAggregateBytes(stageProgress, key, byteLength);
}
emitStageUpdate('progress', stage);
cb(null, chunk);
},
flush(cb) {
stageProgress.count += 1;
if (key) {
incrementAggregateCount(stageProgress, key);
}
emitStageUpdate('progress', stage);
cb(null);
},
});
asset.stream.on('error', (err: Error) => progressTransform.destroy(err));
asset.stream.pipe(progressTransform);
asset.stream = progressTransform;
callback(null, asset);
},
});
}
#updateAggregateBytes = (stageProgress: StageProgress, key: string, bytes: number) => {
if (!stageProgress.aggregates) {
stageProgress.aggregates = {};
}
if (!stageProgress.aggregates[key]) {
stageProgress.aggregates[key] = { count: 0, bytes: 0 };
}
stageProgress.aggregates[key].bytes += bytes;
};
#incrementAggregateCount = (stageProgress: StageProgress, key: string) => {
if (!stageProgress.aggregates) {
stageProgress.aggregates = {};
}
if (!stageProgress.aggregates[key]) {
stageProgress.aggregates[key] = { count: 0, bytes: 0 };
}
stageProgress.aggregates[key].count += 1;
};
/**
* Shorthand method used to trigger transfer update events to every listeners
*/
@@ -589,6 +673,7 @@ class TransferEngine<
if (!destination.destroyed) {
destination.destroy(e as Error);
}
throw e;
} finally {
updateEndTime();
}
@@ -916,14 +1001,41 @@ class TransferEngine<
const destination = await this.destinationProvider.createAssetsWriteStream?.();
const transform = this.#createStageTransformStream(stage);
const tracker = this.#progressTracker(stage, {
size: (value: IAsset) => value.stats.size,
const tracker = this.#progressTrackerChunks(stage, {
key: (value: IAsset) => extname(value.filename) || 'No extension',
});
await this.#mergeSourceStageTotals(stage);
await this.#transferStage({ stage, source, destination, transform, tracker });
}
/**
* Merge optional source-reported totals into progress before the stage starts (CLI ETA / totals).
*/
async #mergeSourceStageTotals(stage: TransferStage) {
const getTotals = this.sourceProvider.getStageTotals;
if (!getTotals) {
return;
}
const totals = await getTotals.call(this.sourceProvider, stage);
if (!totals || (totals.totalBytes == null && totals.totalCount == null)) {
return;
}
if (!this.progress.data[stage]) {
this.progress.data[stage] = { count: 0, bytes: 0, startTime: Date.now() };
}
const stageProgress = this.progress.data[stage];
if (!stageProgress) {
return;
}
if (totals.totalBytes != null) {
stageProgress.totalBytes = totals.totalBytes;
}
if (totals.totalCount != null) {
stageProgress.totalCount = totals.totalCount;
}
}
async transferConfiguration(): Promise<void> {
const stage: TransferStage = 'configuration';
if (this.shouldSkipStage(stage)) {
@@ -6,14 +6,19 @@ jest.mock('fs');
import fs from 'fs-extra';
import { Writable } from 'stream-chain';
import { createLocalFileDestinationProvider, ILocalFileDestinationProviderOptions } from '..';
import {
assertWriteStreamBackpressure,
createSlowWritable,
} from '../../../../__tests__/test-utils';
import * as encryption from '../../../../utils/encryption';
import { createFilePathFactory, createTarEntryStream } from '../utils';
fs.createWriteStream = jest.fn().mockReturnValue(
new Writable({
objectMode: true,
write() {},
})
fs.createWriteStream = jest.fn().mockImplementation(
() =>
new Writable({
objectMode: true,
write() {},
})
);
const filePath = './test-file';
@@ -225,4 +230,31 @@ describe('Local File Destination Provider', () => {
expect(configurationStream instanceof stream.Writable).toBeTruthy();
});
});
describe('Backpressure', () => {
it('entities write stream applies backpressure when downstream is slow', async () => {
const { writable: slowWritable } = createSlowWritable<string>({
objectMode: true,
highWaterMark: 1,
delayMs: 12,
});
(createTarEntryStream as jest.Mock).mockImplementation(() => slowWritable);
const provider = createLocalFileDestinationProvider({
encryption: { enabled: false },
compression: { enabled: false },
file: { path: filePath },
});
await provider.bootstrap();
const writeStream = provider.createEntitiesWriteStream();
const chunks = Array.from({ length: 25 }, (_, i) => ({ id: i, title: `Item ${i}` }));
const { sourcePaused } = await assertWriteStreamBackpressure(writeStream, chunks, {
delayMs: 12,
});
expect(sourcePaused).toBe(true);
}, 5000);
});
});
@@ -1,11 +1,16 @@
import { Readable } from 'stream';
import path from 'path';
import os from 'os';
import fs from 'fs-extra';
import tarStream from 'tar-stream';
import { pipeline } from 'stream/promises';
import type { ILocalFileSourceProviderOptions } from '..';
import { createLocalFileSourceProvider } from '..';
import { isFilePathInDirname, isPathEquivalent, unknownPathToPosix } from '../utils';
import { assertReadStreamBackpressure } from '../../../../__tests__/test-utils';
describe('File source provider', () => {
test('returns assets stream', () => {
test('exposes createAssetsReadStream (starting the stream opens the backup file on disk)', () => {
const options: ILocalFileSourceProviderOptions = {
file: {
path: './test-file',
@@ -18,9 +23,7 @@ describe('File source provider', () => {
},
};
const provider = createLocalFileSourceProvider(options);
const stream = provider.createAssetsReadStream();
expect(stream instanceof Readable).toBeTruthy();
expect(provider.createAssetsReadStream).toEqual(expect.any(Function));
});
describe('utils', () => {
@@ -130,4 +133,47 @@ describe('File source provider', () => {
}
);
});
describe('Backpressure', () => {
test('entities read stream pauses under backpressure when reading from tar', async () => {
const tmpDir = os.tmpdir();
const tarPath = path.join(tmpDir, `strapi-dt-backpressure-${Date.now()}.tar`);
const pack = tarStream.pack();
pack.entry(
{ name: 'metadata.json' },
JSON.stringify({
createdAt: new Date().toISOString(),
strapi: { version: '1.0.0' },
})
);
const entityLines = Array.from(
{ length: 25 },
(_, i) => `${JSON.stringify({ uid: 'api::foo.foo', id: i + 1, title: `Entity ${i}` })}\n`
).join('');
pack.entry({ name: 'entities/entities_00000.jsonl' }, entityLines);
pack.finalize();
const writeStream = fs.createWriteStream(tarPath);
await pipeline(pack, writeStream);
const provider = createLocalFileSourceProvider({
file: { path: tarPath },
compression: { enabled: false },
encryption: { enabled: false },
});
await provider.bootstrap();
const stream = provider.createEntitiesReadStream();
const { sourcePaused, chunks } = await assertReadStreamBackpressure(stream, {
delayMs: 12,
minChunksForBackpressure: 10,
});
await fs.remove(tarPath).catch(() => {});
expect(sourcePaused).toBe(true);
expect(chunks.length).toBe(25);
}, 8000);
});
});
@@ -15,6 +15,7 @@ import type { IAsset, IMetadata, ISourceProvider, ProviderType, IFile } from '..
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
import * as utils from '../../../utils';
import { write } from '../../../utils/writable-async-write';
import { ProviderInitializationError, ProviderTransferError } from '../../../errors/providers';
import { isFilePathInDirname, isPathEquivalent, unknownPathToPosix } from './utils';
@@ -79,6 +80,29 @@ class LocalFileSourceProvider implements ISourceProvider {
});
}
/**
* tar `Parser` invokes the pipeline completion callback when the archive ends, but it does not
* reliably await async `onReadEntry` — defer `end` until outstanding async entry work is done.
*/
#endPassThroughWhenTarIdle(
outStream: PassThrough,
activeAsyncEntries: () => number,
err?: Error | null
) {
if (err) {
outStream.destroy(err);
return;
}
const tick = () => {
if (activeAsyncEntries() === 0) {
outStream.end();
} else {
setImmediate(tick);
}
};
tick();
}
/**
* Pre flight checks regarding the provided options, making sure that the file can be opened (decrypted, decompressed), etc.
*/
@@ -167,6 +191,16 @@ class LocalFileSourceProvider implements ISourceProvider {
const loadAssetMetadata = this.#loadAssetMetadata.bind(this);
this.#reportInfo('creating assets read stream');
let activeAsyncEntries = 0;
const runReadEntry = async (fn: () => Promise<void>) => {
activeAsyncEntries += 1;
try {
await fn();
} finally {
activeAsyncEntries -= 1;
}
};
pipeline(
[
inStream,
@@ -179,27 +213,29 @@ class LocalFileSourceProvider implements ISourceProvider {
return isFilePathInDirname('assets/uploads', filePath);
},
async onReadEntry(entry: ReadEntry) {
const { path: filePath, size = 0 } = entry;
const normalizedPath = unknownPathToPosix(filePath);
const file = path.basename(normalizedPath);
let metadata;
try {
metadata = await loadAssetMetadata(`assets/metadata/${file}.json`);
} catch (error) {
throw new Error(`Failed to read metadata for ${file}`);
}
const asset: IAsset = {
metadata,
filename: file,
filepath: normalizedPath,
stats: { size },
stream: entry as unknown as Readable,
};
outStream.write(asset);
await runReadEntry(async () => {
const { path: filePath, size = 0 } = entry;
const normalizedPath = unknownPathToPosix(filePath);
const file = path.basename(normalizedPath);
let metadata;
try {
metadata = await loadAssetMetadata(`assets/metadata/${file}.json`);
} catch (error) {
throw new Error(`Failed to read metadata for ${file}`);
}
const asset: IAsset = {
metadata,
filename: file,
filepath: normalizedPath,
stats: { size },
stream: entry as unknown as Readable,
};
await write(outStream, asset);
});
},
}),
],
() => outStream.end()
(err) => this.#endPassThroughWhenTarIdle(outStream, () => activeAsyncEntries, err)
);
return outStream;
@@ -233,6 +269,16 @@ class LocalFileSourceProvider implements ISourceProvider {
const outStream = new PassThrough({ objectMode: true });
let activeAsyncEntries = 0;
const runReadEntry = async (fn: () => Promise<void>) => {
activeAsyncEntries += 1;
try {
await fn();
} finally {
activeAsyncEntries -= 1;
}
};
pipeline(
[
inStream,
@@ -246,43 +292,41 @@ class LocalFileSourceProvider implements ISourceProvider {
},
async onReadEntry(entry: ReadEntry) {
const transforms = [
// JSONL parser to read the data chunks one by one (line by line)
parser({
checkErrors: true,
}),
// The JSONL parser returns each line as key/value
(line: { key: string; value: object }) => line.value,
];
await runReadEntry(async () => {
const transforms = [
// JSONL parser to read the data chunks one by one (line by line)
parser({
checkErrors: true,
}),
// The JSONL parser returns each line as key/value
(line: { key: string; value: object }) => line.value,
];
const stream = entry.pipe(chain(transforms));
const stream = entry.pipe(chain(transforms));
try {
for await (const chunk of stream) {
outStream.write(chunk);
try {
for await (const chunk of stream) {
await write(outStream, chunk);
}
} catch (e: unknown) {
outStream.destroy(
new ProviderTransferError(
`Error parsing backup files from backup file ${entry.path}: ${
(e as Error).message
}`,
{
details: {
error: e,
},
}
)
);
}
} catch (e: unknown) {
outStream.destroy(
new ProviderTransferError(
`Error parsing backup files from backup file ${entry.path}: ${
(e as Error).message
}`,
{
details: {
error: e,
},
}
)
);
}
});
},
}),
],
async () => {
// Manually send the 'end' event to the out stream
// once every entry has finished streaming its content
outStream.end();
}
(err) => this.#endPassThroughWhenTarIdle(outStream, () => activeAsyncEntries, err)
);
return outStream;
@@ -0,0 +1,131 @@
import { Readable } from 'stream';
import type { Core } from '@strapi/types';
import type { IAsset } from '../../../../../types';
import { createTransaction } from '../../../../utils/transaction';
import { createAssetsDestinationWritable } from '../assets-destination-writable';
/**
* Push may send start + chunks + end in one batch; `write()` must return before `uploadStream`
* resolves so the PassThrough is not read-only until upload completes (avoids assets-step deadlock).
*/
describe('createAssetsDestinationWritable (push transfer regression)', () => {
test('invokes write callback before uploadStream completes (PassThrough can be fed in the same batch)', async () => {
let releaseUpload!: (value?: unknown) => void;
const uploadBlocked = new Promise((resolve) => {
releaseUpload = resolve;
});
let uploadFinished = false;
const uploadStream = jest.fn(async () => {
await uploadBlocked;
uploadFinished = true;
});
const mockFindOne = jest.fn().mockResolvedValue({ id: 1, url: 'x.jpg', formats: {} });
const mockUpdate = jest.fn().mockResolvedValue(null);
const strapi = {
config: {
get(service: string) {
if (service === 'plugin::upload') {
return { provider: 'local' };
}
return {};
},
},
db: {
transaction(fn: (arg: { trx: object; rollback: () => Promise<void> }) => Promise<void>) {
fn({ trx: {}, rollback: async () => Promise.resolve() });
return Promise.resolve();
},
query: jest.fn(() => ({
findOne: mockFindOne,
update: mockUpdate,
})),
},
plugin(name: string) {
if (name === 'upload') {
return {
provider: { uploadStream },
};
}
return {};
},
} as unknown as Core.Strapi;
const transaction = createTransaction(strapi);
const stream = createAssetsDestinationWritable({
strapi,
transaction,
resolveUploadFileId: () => 1,
restoreMediaEntitiesContent: false,
removeAssetsBackup: async () => Promise.resolve(),
});
const file: IAsset = {
filename: 'a.jpg',
filepath: '/a',
stats: { size: 10 },
stream: Readable.from([Buffer.from('hello')]),
metadata: {
hash: 'h',
name: 'a',
id: 1,
url: 'a.jpg',
size: 10,
mime: 'image/jpeg',
},
};
const writeSettled = new Promise<void>((resolve, reject) => {
stream.write(file, (err) => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
let timeoutId: ReturnType<typeof setTimeout> | undefined;
try {
await Promise.race([
writeSettled,
new Promise<void>((_, reject) => {
timeoutId = setTimeout(
() =>
reject(
new Error(
'Timed out waiting for assets Writable write() callback (uploadStream must not block the write callback when start and chunks arrive in one batch)'
)
),
200
);
}),
]);
} finally {
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
}
}
expect(uploadStream).toHaveBeenCalled();
expect(uploadFinished).toBe(false);
releaseUpload();
await new Promise<void>((resolve) => {
setImmediate(resolve);
});
expect(uploadFinished).toBe(true);
await new Promise<void>((resolve, reject) => {
stream.end((err?: Error | null) => (err ? reject(err) : resolve()));
});
transaction.end();
});
});
@@ -1,11 +1,14 @@
import { createLocalStrapiDestinationProvider } from '../index';
import * as restoreApi from '../strategies/restore';
import {
assertWriteStreamBackpressure,
createSlowWritable,
getStrapiFactory,
getContentTypes,
setGlobalStrapi,
getStrapiModels,
} from '../../../../__tests__/test-utils';
import type { IEntity } from '../../../../../types';
afterEach(() => {
jest.clearAllMocks();
@@ -253,4 +256,42 @@ describe('Local Strapi Source Destination', () => {
expect(deleteAllSpy).toBeCalledTimes(1);
});
});
describe('Backpressure', () => {
test('entities write stream applies backpressure to slow down fast readables', async () => {
const { writable: slowWritable } = createSlowWritable<IEntity>({
highWaterMark: 1,
delayMs: 15,
});
jest
.spyOn(restoreApi, 'createEntitiesWriteStream')
.mockReturnValue(slowWritable as ReturnType<typeof restoreApi.createEntitiesWriteStream>);
const provider = createLocalStrapiDestinationProvider({
getStrapi: getStrapiFactory({
db: {
transaction,
lifecycles: { enable: jest.fn(), disable: jest.fn() },
},
...strapiCommonProperties,
}),
strategy: 'restore',
restore: { entities: { exclude: [] } },
});
await provider.bootstrap();
const writeStream = provider.createEntitiesWriteStream();
const entityChunks: IEntity[] = Array.from({ length: 25 }, (_, i) => ({
id: i + 1,
type: 'api::foo.foo',
data: { title: `Entity ${i}`, documentId: `doc-${i}` },
}));
const { sourcePaused } = await assertWriteStreamBackpressure(writeStream, entityChunks, {
delayMs: 15,
});
expect(sourcePaused).toBe(true);
}, 5000);
});
});
@@ -0,0 +1,128 @@
import { Writable, Readable } from 'stream';
import type { Core } from '@strapi/types';
import type { IAsset, IFile } from '../../../../types';
import type { Transaction } from '../../../../types/utils';
export interface CreateAssetsDestinationWritableOptions {
strapi: Core.Strapi;
transaction: Transaction;
resolveUploadFileId: (metadata: { id: number }) => number | undefined;
restoreMediaEntitiesContent: boolean;
removeAssetsBackup: () => Promise<void>;
}
/**
* Writable for restoring upload assets during a local push destination transfer.
*
* The Writable `write()` callback must return as soon as the chunk is accepted — **before**
* `uploadStream` finishes — so the remote push handler can feed PassThrough data in the same
* WebSocket batch after an asset `start` row (see `streamAsset` in remote `push` handler).
*/
export function createAssetsDestinationWritable(
options: CreateAssetsDestinationWritableOptions
): Writable {
const {
strapi,
transaction,
resolveUploadFileId,
restoreMediaEntitiesContent,
removeAssetsBackup,
} = options;
let pendingUploads = 0;
return new Writable({
objectMode: true,
async final(next) {
while (pendingUploads > 0) {
await new Promise<void>((resolve) => {
setImmediate(resolve);
});
}
await removeAssetsBackup();
next();
},
write(chunk: IAsset, _encoding, callback) {
const uploadData = {
...chunk.metadata,
stream: Readable.from(chunk.stream),
buffer: chunk?.buffer,
};
const provider = strapi.config.get<{ provider: string }>('plugin::upload').provider;
const fileId = resolveUploadFileId(uploadData);
if (!fileId) {
callback(new Error(`File ID not found for ID: ${uploadData.id}`));
return;
}
if (!transaction) {
callback(new Error('Transaction not available for asset upload'));
return;
}
pendingUploads += 1;
transaction
.attach(async () => {
try {
await strapi.plugin('upload').provider.uploadStream(uploadData);
if (!restoreMediaEntitiesContent) {
return;
}
if (uploadData?.type) {
const entry: IFile = await strapi.db.query('plugin::upload.file').findOne({
where: { id: fileId },
});
if (!entry) {
throw new Error('file not found');
}
const specificFormat = entry?.formats?.[uploadData.type];
if (specificFormat) {
specificFormat.url = uploadData.url;
}
await strapi.db.query('plugin::upload.file').update({
where: { id: entry.id },
data: {
formats: entry.formats,
provider,
},
});
return;
}
const entry: IFile = await strapi.db.query('plugin::upload.file').findOne({
where: { id: fileId },
});
if (!entry) {
throw new Error('file not found');
}
entry.url = uploadData.url;
await strapi.db.query('plugin::upload.file').update({
where: { id: entry.id },
data: {
url: entry.url,
provider,
},
});
} catch (error) {
throw new Error(`Error while uploading asset ${chunk.filename} ${error}`);
}
})
.finally(() => {
pendingUploads -= 1;
})
.catch((error: unknown) => {
const err = error instanceof Error ? error : new Error(String(error));
process.nextTick(() => {
this.destroy(err);
});
});
callback();
},
});
}
@@ -3,16 +3,10 @@ import path from 'path';
import * as fse from 'fs-extra';
import type { Knex } from 'knex';
import type { Core, Struct } from '@strapi/types';
import type {
IAsset,
IDestinationProvider,
IFile,
IMetadata,
ProviderType,
Transaction,
} from '../../../../types';
import type { IDestinationProvider, IMetadata, ProviderType, Transaction } from '../../../../types';
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
import { createAssetsDestinationWritable } from './assets-destination-writable';
import { restore } from './strategies';
import * as utils from '../../../utils';
import {
@@ -316,85 +310,14 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
);
}
const removeAssetsBackup = this.#removeAssetsBackup.bind(this);
const strapi = this.strapi;
const transaction = this.transaction;
const fileEntitiesMapper = this.#entitiesMapper['plugin::upload.file'];
const restoreMediaEntitiesContent = this.#isContentTypeIncluded('plugin::upload.file');
return new Writable({
objectMode: true,
async final(next) {
// Delete the backup folder
await removeAssetsBackup();
next();
},
async write(chunk: IAsset, _encoding, callback) {
await transaction?.attach(async () => {
const uploadData = {
...chunk.metadata,
stream: Readable.from(chunk.stream),
buffer: chunk?.buffer,
};
const provider = strapi.config.get<{ provider: string }>('plugin::upload').provider;
const fileId = fileEntitiesMapper?.[uploadData.id];
if (!fileId) {
return callback(new Error(`File ID not found for ID: ${uploadData.id}`));
}
try {
await strapi.plugin('upload').provider.uploadStream(uploadData);
// if we're not supposed to transfer the associated entities, stop here
if (!restoreMediaEntitiesContent) {
return callback();
}
// Files formats are stored within the parent file entity
if (uploadData?.type) {
const entry: IFile = await strapi.db.query('plugin::upload.file').findOne({
where: { id: fileId },
});
if (!entry) {
throw new Error('file not found');
}
const specificFormat = entry?.formats?.[uploadData.type];
if (specificFormat) {
specificFormat.url = uploadData.url;
}
await strapi.db.query('plugin::upload.file').update({
where: { id: entry.id },
data: {
formats: entry.formats,
provider,
},
});
return callback();
}
const entry: IFile = await strapi.db.query('plugin::upload.file').findOne({
where: { id: fileId },
});
if (!entry) {
throw new Error('file not found');
}
entry.url = uploadData.url;
await strapi.db.query('plugin::upload.file').update({
where: { id: entry.id },
data: {
url: entry.url,
provider,
},
});
return callback();
} catch (error) {
return callback(new Error(`Error while uploading asset ${chunk.filename} ${error}`));
}
});
},
return createAssetsDestinationWritable({
strapi: this.strapi,
transaction: this.transaction!,
resolveUploadFileId: (metadata) => fileEntitiesMapper?.[metadata.id],
restoreMediaEntitiesContent: this.#isContentTypeIncluded('plugin::upload.file'),
removeAssetsBackup: this.#removeAssetsBackup.bind(this),
});
}
@@ -0,0 +1,58 @@
import { Readable } from 'stream';
import { createAssetsStream } from '../assets';
describe('Local source assets stream warnings', () => {
test('reports warning callback when media DB row points to missing file', async () => {
const missingFile = {
id: 42,
hash: 'missing-hash',
ext: '.jpg',
url: '/uploads/does-not-exist.jpg',
provider: 'local',
formats: undefined,
};
const warn = jest.fn();
const strapi = {
db: {
queryBuilder: jest.fn(() => ({
select: jest.fn().mockReturnThis(),
stream: jest.fn(() => Readable.from([missingFile])),
})),
},
dirs: {
static: { public: '/tmp/this-path-should-not-exist-for-test' },
},
log: {
warn,
},
plugins: {
upload: {
provider: {
isPrivate: jest.fn().mockResolvedValue(false),
},
},
},
config: {
get: jest.fn(() => ({ provider: 'local' })),
},
} as any;
const onWarning = jest.fn();
const stream = createAssetsStream(strapi, { onWarning });
const items: unknown[] = [];
for await (const item of stream) {
items.push(item);
}
expect(items).toHaveLength(0);
expect(onWarning).toHaveBeenCalledWith(
expect.stringContaining(
'Media item 42 (hash: missing-hash) exists in database but no corresponding file was found to transfer'
)
);
expect(warn).toHaveBeenCalledWith(expect.stringContaining('Media item 42'));
});
});
@@ -0,0 +1,183 @@
import { Readable } from 'stream';
import * as assets from '../assets';
import { estimateAssetTotals } from '../estimate-asset-totals';
jest.mock('../assets', () => {
const actual = jest.requireActual('../assets');
return {
__esModule: true,
...actual,
getFileStatsForTransfer: jest.fn(),
signUploadFileForTransfer: jest.fn().mockResolvedValue(undefined),
};
});
describe('estimateAssetTotals', () => {
const getStrapiMock = (rows: Record<string, unknown>[]) => ({
dirs: { static: { public: '/data/public' } },
log: { warn: jest.fn() },
db: {
queryBuilder: jest.fn(() => ({
select: jest.fn().mockReturnThis(),
stream: jest.fn(() => Readable.from(rows)),
})),
},
plugins: {
upload: {
provider: {
isPrivate: jest.fn().mockResolvedValue(false),
},
},
},
config: {
get: jest.fn(() => ({ provider: 'local' })),
},
});
beforeEach(() => {
jest.mocked(assets.getFileStatsForTransfer).mockReset();
jest.mocked(assets.signUploadFileForTransfer).mockClear();
});
test('sums sizes and counts for local files (no formats)', async () => {
jest.mocked(assets.getFileStatsForTransfer).mockResolvedValue({ size: 50 });
const strapi = getStrapiMock([
{
provider: 'local',
url: '/uploads/a.png',
hash: 'h1',
ext: '.png',
id: 1,
},
{
provider: 'local',
url: '/uploads/b.png',
hash: 'h2',
ext: '.png',
id: 2,
},
]);
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
totalBytes: 100,
totalCount: 2,
});
expect(assets.getFileStatsForTransfer).toHaveBeenCalledTimes(2);
});
test('includes format rows in totals', async () => {
jest
.mocked(assets.getFileStatsForTransfer)
.mockResolvedValueOnce({ size: 100 })
.mockResolvedValueOnce({ size: 30 });
const strapi = getStrapiMock([
{
provider: 'local',
url: '/uploads/main.jpg',
hash: 'm',
ext: '.jpg',
id: 1,
formats: {
thumbnail: {
url: '/uploads/thumb.jpg',
hash: 't',
ext: '.jpg',
},
},
},
]);
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
totalBytes: 130,
totalCount: 2,
});
});
test('remote files use DB sizes only when main and formats have size (no HTTP stat)', async () => {
const strapi = getStrapiMock([
{
provider: 'aws-s3',
url: 'https://bucket/a.png',
hash: 'h1',
ext: '.png',
id: 1,
size: 400,
formats: {
thumbnail: {
url: 'https://bucket/a_thumb.png',
hash: 't',
ext: '.png',
size: 50,
},
},
},
{
provider: 'aws-s3',
url: 'https://bucket/b.png',
hash: 'h2',
ext: '.png',
id: 2,
size: 100,
},
]);
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
totalBytes: 550,
totalCount: 3,
});
expect(assets.getFileStatsForTransfer).not.toHaveBeenCalled();
expect(assets.signUploadFileForTransfer).not.toHaveBeenCalled();
});
test('remote files fall back to HTTP stat when size is missing on main', async () => {
jest.mocked(assets.getFileStatsForTransfer).mockResolvedValue({ size: 999 });
const strapi = getStrapiMock([
{
provider: 'aws-s3',
url: 'https://bucket/a.png',
hash: 'h1',
ext: '.png',
id: 1,
},
]);
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
totalBytes: 999,
totalCount: 1,
});
expect(assets.signUploadFileForTransfer).toHaveBeenCalledTimes(1);
expect(assets.getFileStatsForTransfer).toHaveBeenCalledTimes(1);
});
test('remote files fall back to HTTP stat when a format size is missing', async () => {
jest.mocked(assets.getFileStatsForTransfer).mockResolvedValueOnce({ size: 12 }); // thumbnail probe only
const strapi = getStrapiMock([
{
provider: 'aws-s3',
url: 'https://bucket/a.png',
hash: 'h1',
ext: '.png',
id: 1,
size: 400,
formats: {
thumbnail: {
url: 'https://bucket/a_thumb.png',
hash: 't',
ext: '.png',
},
},
},
]);
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
totalBytes: 412,
totalCount: 2,
});
expect(assets.signUploadFileForTransfer).toHaveBeenCalledTimes(1);
expect(assets.getFileStatsForTransfer).toHaveBeenCalledTimes(1);
});
});
@@ -2,6 +2,7 @@ import { Readable } from 'stream';
import type { IEntity } from '../../../../../types';
import {
assertReadStreamBackpressure,
collect,
createMockedQueryBuilder,
getStrapiFactory,
@@ -258,4 +259,69 @@ describe('Local Strapi Source Provider', () => {
]);
});
});
describe('Backpressure', () => {
test('entities read stream pauses under backpressure and preserves data', async () => {
const itemCount = 25;
const contentTypes = {
foo: { uid: 'foo', attributes: { title: { type: 'string' } } },
};
const queryBuilder = createMockedQueryBuilder({
foo: Array.from({ length: itemCount }, (_, i) => ({ id: i + 1, title: `Title ${i}` })),
});
const provider = createLocalStrapiSourceProvider({
getStrapi: getStrapiFactory({
contentTypes,
db: {
queryBuilder,
lifecycles: { enable: jest.fn(), disable: jest.fn() },
},
getModel: jest.fn((uid: string) => contentTypes[uid as keyof typeof contentTypes]),
}),
});
await provider.bootstrap();
const stream = (await provider.createEntitiesReadStream()) as Readable;
const { sourcePaused, chunks } = await assertReadStreamBackpressure<IEntity>(stream, {
delayMs: 12,
minChunksForBackpressure: 10,
});
expect(sourcePaused).toBe(true);
expect(chunks).toHaveLength(itemCount);
chunks.forEach((entity) => {
expect(entity).toMatchObject({
type: 'foo',
id: expect.any(Number),
data: expect.any(Object),
});
});
}, 5000);
test('schemas read stream pauses under backpressure', async () => {
const contentTypes = {
foo: { uid: 'foo', attributes: {} },
bar: { uid: 'bar', attributes: {} },
baz: { uid: 'baz', attributes: {} },
};
const components = {};
const provider = createLocalStrapiSourceProvider({
getStrapi: getStrapiFactory({
contentTypes,
components,
db: { lifecycles: { enable: jest.fn(), disable: jest.fn() } },
}),
});
await provider.bootstrap();
const stream = provider.createSchemasReadStream();
const { sourcePaused, chunks } = await assertReadStreamBackpressure(stream, {
delayMs: 12,
});
expect(sourcePaused).toBe(true);
expect(chunks).toHaveLength(3);
}, 5000);
});
});
@@ -41,7 +41,7 @@ function getFileStream(
return readableStream;
}
function getFileStats(
export function getFileStatsForTransfer(
filepath: string,
strapi: Core.Strapi,
isLocal = false
@@ -71,19 +71,17 @@ function getFileStats(
});
}
async function signFile(file: IFile) {
export async function signUploadFileForTransfer(strapi: Core.Strapi, file: IFile) {
const { provider } = strapi.plugins.upload;
const { provider: providerName } = strapi.config.get('plugin.upload') as { provider: string };
const isPrivate = await provider.isPrivate();
if (file?.provider === providerName && isPrivate) {
const signUrl = async (file: IFile) => {
const signedUrl = await provider.getSignedUrl(file);
file.url = signedUrl.url;
const signUrl = async (f: IFile) => {
const signedUrl = await provider.getSignedUrl(f);
f.url = signedUrl.url;
};
// Sign the original file
await signUrl(file);
// Sign each file format
if (file.formats) {
for (const format of Object.keys(file.formats)) {
await signUrl(file.formats[format]);
@@ -92,10 +90,23 @@ async function signFile(file: IFile) {
}
}
const missingAssetWarningMessage = (file: IFile, filepath: string, format?: string): string => {
const formatPart = format ? ` (format: ${format})` : '';
return `[Data transfer] Media item ${file.id} (hash: ${file.hash}) exists in database but no corresponding file was found to transfer${formatPart}. Path: ${filepath}`;
};
/**
* Generate and consume assets streams in order to stream each file individually
*/
export const createAssetsStream = (strapi: Core.Strapi): Duplex => {
export const createAssetsStream = (
strapi: Core.Strapi,
options: { onWarning?: (message: string) => void } = {}
): Duplex => {
const warnMissingAsset = (message: string) => {
strapi.log.warn(message);
options.onWarning?.(message);
};
const generator: () => AsyncGenerator<IAsset, void> = async function* () {
const stream: Readable = strapi.db
.queryBuilder('plugin::upload.file')
@@ -108,10 +119,23 @@ export const createAssetsStream = (strapi: Core.Strapi): Duplex => {
for await (const file of stream) {
const isLocalProvider = file.provider === 'local';
if (!isLocalProvider) {
await signFile(file);
await signUploadFileForTransfer(strapi, file);
}
const filepath = isLocalProvider ? join(strapi.dirs.static.public, file.url) : file.url;
const stats = await getFileStats(filepath, strapi, isLocalProvider);
let stats: { size: number };
try {
stats = await getFileStatsForTransfer(filepath, strapi, isLocalProvider);
} catch (err: unknown) {
const code =
err && typeof err === 'object' && 'code' in err
? (err as NodeJS.ErrnoException).code
: undefined;
if (code === 'ENOENT') {
warnMissingAsset(missingAssetWarningMessage(file, filepath));
continue;
}
throw err;
}
const stream = getFileStream(filepath, strapi, isLocalProvider);
yield {
@@ -128,7 +152,24 @@ export const createAssetsStream = (strapi: Core.Strapi): Duplex => {
const fileFormatFilepath = isLocalProvider
? join(strapi.dirs.static.public, fileFormat.url)
: fileFormat.url;
const fileFormatStats = await getFileStats(fileFormatFilepath, strapi, isLocalProvider);
let fileFormatStats: { size: number };
try {
fileFormatStats = await getFileStatsForTransfer(
fileFormatFilepath,
strapi,
isLocalProvider
);
} catch (err: unknown) {
const code =
err && typeof err === 'object' && 'code' in err
? (err as NodeJS.ErrnoException).code
: undefined;
if (code === 'ENOENT') {
warnMissingAsset(missingAssetWarningMessage(file, fileFormatFilepath, format));
continue;
}
throw err;
}
const fileFormatStream = getFileStream(fileFormatFilepath, strapi, isLocalProvider);
const metadata = { ...fileFormat, type: format, id: file.id, mainHash: file.hash };
yield {
@@ -0,0 +1,161 @@
import { join } from 'path';
import type { Readable } from 'stream';
import type { Core } from '@strapi/types';
import type { IAsset, StageTotalsEstimate } from '../../../../types';
import { getFileStatsForTransfer, signUploadFileForTransfer } from './assets';
type UploadFileRecord = IAsset['metadata'];
/** Strapi stores byte size on each file record; use for remote totals to avoid per-URL HTTP. */
function hasReliableDbSize(size: unknown): size is number {
return typeof size === 'number' && Number.isFinite(size) && size >= 0;
}
/** When every main + format has a DB size, remote rows need no signing or HTTP stat. */
function remoteRowCanUseDbOnly(file: UploadFileRecord): boolean {
if (!hasReliableDbSize(file.size)) {
return false;
}
if (!file.formats) {
return true;
}
for (const key of Object.keys(file.formats)) {
if (!hasReliableDbSize(file.formats[key].size)) {
return false;
}
}
return true;
}
/**
* Sum sizes and counts for the same asset rows `createAssetsStream` would yield (main + formats),
* skipping missing files with ENOENT like the stream does. Used for transfer progress totals / ETA.
*
* - **Local (`provider === 'local'`):** `stat` on disk (source of truth; matches ENOENT skips).
* - **Remote:** sum `size` from DB when present on main and every format; otherwise sign + `fetch` / `Content-Length` like before.
*/
export async function estimateAssetTotals(strapi: Core.Strapi): Promise<StageTotalsEstimate> {
let totalBytes = 0;
let totalCount = 0;
const stream: Readable = strapi.db.queryBuilder('plugin::upload.file').select('*').stream();
for await (const file of stream) {
const isLocalProvider = file.provider === 'local';
if (isLocalProvider) {
const filepath = join(strapi.dirs.static.public, file.url);
try {
const stats = await getFileStatsForTransfer(filepath, strapi, true);
totalBytes += stats.size;
totalCount += 1;
} catch (err: unknown) {
const code =
err && typeof err === 'object' && 'code' in err
? (err as NodeJS.ErrnoException).code
: undefined;
if (code === 'ENOENT') {
strapi.log.warn(`[Data transfer] Skipping missing asset file: ${filepath}`);
continue;
}
throw err;
}
if (file.formats) {
for (const format of Object.keys(file.formats)) {
const fileFormat = file.formats[format];
const fileFormatFilepath = join(strapi.dirs.static.public, fileFormat.url);
try {
const fileFormatStats = await getFileStatsForTransfer(fileFormatFilepath, strapi, true);
totalBytes += fileFormatStats.size;
totalCount += 1;
} catch (err: unknown) {
const code =
err && typeof err === 'object' && 'code' in err
? (err as NodeJS.ErrnoException).code
: undefined;
if (code === 'ENOENT') {
strapi.log.warn(`[Data transfer] Skipping missing asset file: ${fileFormatFilepath}`);
continue;
}
throw err;
}
}
}
continue;
}
// Remote: prefer DB sizes (fast); fall back to signed URL + HTTP where `size` is missing.
if (remoteRowCanUseDbOnly(file)) {
totalBytes += file.size;
totalCount += 1;
if (file.formats) {
for (const format of Object.keys(file.formats)) {
totalBytes += file.formats[format].size;
totalCount += 1;
}
}
continue;
}
await signUploadFileForTransfer(strapi, file);
if (hasReliableDbSize(file.size)) {
totalBytes += file.size;
totalCount += 1;
} else {
try {
const stats = await getFileStatsForTransfer(file.url, strapi, false);
totalBytes += stats.size;
totalCount += 1;
} catch (err: unknown) {
const code =
err && typeof err === 'object' && 'code' in err
? (err as NodeJS.ErrnoException).code
: undefined;
if (code === 'ENOENT') {
strapi.log.warn(`[Data transfer] Skipping missing asset file: ${file.url}`);
continue;
}
throw err;
}
}
if (file.formats) {
for (const format of Object.keys(file.formats)) {
const fileFormat = file.formats[format];
const fileFormatFilepath = fileFormat.url;
if (hasReliableDbSize(fileFormat.size)) {
totalBytes += fileFormat.size;
totalCount += 1;
} else {
try {
const fileFormatStats = await getFileStatsForTransfer(
fileFormatFilepath,
strapi,
false
);
totalBytes += fileFormatStats.size;
totalCount += 1;
} catch (err: unknown) {
const code =
err && typeof err === 'object' && 'code' in err
? (err as NodeJS.ErrnoException).code
: undefined;
if (code === 'ENOENT') {
strapi.log.warn(`[Data transfer] Skipping missing asset file: ${fileFormatFilepath}`);
continue;
}
throw err;
}
}
}
}
}
return { totalBytes, totalCount };
}
@@ -2,12 +2,13 @@ import { Readable } from 'stream';
import { chain } from 'stream-chain';
import type { Core, Struct } from '@strapi/types';
import type { IMetadata, ISourceProvider, ProviderType } from '../../../../types';
import type { IMetadata, ISourceProvider, ProviderType, TransferStage } from '../../../../types';
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
import { createEntitiesStream, createEntitiesTransformStream } from './entities';
import { createLinksStream } from './links';
import { createConfigurationStream } from './configuration';
import { createAssetsStream } from './assets';
import { estimateAssetTotals } from './estimate-asset-totals';
import * as utils from '../../../utils';
import { assertValidStrapi } from '../../../utils/providers';
@@ -53,6 +54,17 @@ class LocalStrapiSourceProvider implements ISourceProvider {
});
}
#reportWarning(message: string) {
this.#diagnostics?.report({
details: {
createdAt: new Date(),
message,
origin: 'local-source-provider',
},
kind: 'warning',
});
}
/**
* Reports an error to the diagnostic reporter.
*/
@@ -97,7 +109,8 @@ class LocalStrapiSourceProvider implements ISourceProvider {
getMetadata(): IMetadata {
this.#reportInfo('getting metadata');
const strapiVersion = strapi.config.get<string>('info.strapi');
assertValidStrapi(this.strapi);
const strapiVersion = this.strapi.config.get<string>('info.strapi');
const createdAt = new Date().toISOString();
return {
@@ -152,13 +165,25 @@ class LocalStrapiSourceProvider implements ISourceProvider {
assertValidStrapi(this.strapi, 'Not able to stream assets');
this.#reportInfo('creating assets read stream');
const stream = createAssetsStream(this.strapi);
const stream = createAssetsStream(this.strapi, {
onWarning: (message) => this.#reportWarning(message),
});
stream.on('error', (err) => {
this.#handleStreamError('assets', err);
});
return stream;
}
async getStageTotals(stage: TransferStage) {
if (stage !== 'assets') {
return null;
}
assertValidStrapi(this.strapi, 'Not able to estimate asset totals');
return estimateAssetTotals(this.strapi);
}
}
export type ILocalStrapiSourceProvider = InstanceType<typeof LocalStrapiSourceProvider>;
export { estimateAssetTotals } from './estimate-asset-totals';
@@ -0,0 +1,96 @@
import { EventEmitter } from 'events';
import { Readable } from 'stream';
import type { IAsset } from '../../../../../types';
import { createRemoteStrapiDestinationProvider } from '..';
import { connectToWebsocket, createDispatcher } from '../../utils';
jest.mock('../../utils', () => ({
...jest.requireActual('../../utils'),
connectToWebsocket: jest.fn(),
createDispatcher: jest.fn(),
}));
class MockWebSocket extends EventEmitter {
send = jest.fn((_payload: string, cb?: (err?: Error) => void) => cb?.());
close = jest.fn();
}
describe('Remote destination checksum negotiation', () => {
test('warns and continues without checksums when peer does not support checksum negotiation', async () => {
const ws = new MockWebSocket();
(connectToWebsocket as jest.Mock).mockResolvedValue(ws);
const streamBatches: unknown[][] = [];
const dispatcher = {
dispatchCommand: jest.fn().mockResolvedValue({ transferID: 't1' }), // no `checksums` support
dispatchTransferAction: jest.fn().mockResolvedValue(null),
dispatchTransferStep: jest.fn(
async (msg: { action: string; data?: unknown; step?: string }) => {
if (msg.action === 'stream' && msg.step === 'assets' && Array.isArray(msg.data)) {
streamBatches.push(msg.data);
}
if (msg.action === 'start') {
return { ok: true };
}
if (msg.action === 'end') {
return { ok: true, stats: { started: 1, finished: 1 } };
}
return null;
}
),
setTransferProperties: jest.fn(),
dispatch: jest.fn(),
transferID: 't1',
transferKind: 'push' as const,
};
(createDispatcher as jest.Mock).mockReturnValue(dispatcher);
const diagnostics = { report: jest.fn() };
const provider = createRemoteStrapiDestinationProvider({
strategy: 'restore',
restore: {},
url: new URL('http://localhost:1337/admin'),
verifyChecksums: true,
});
await provider.bootstrap(diagnostics as any);
expect(diagnostics.report).toHaveBeenCalledWith(
expect.objectContaining({
kind: 'warning',
details: expect.objectContaining({
message: expect.stringContaining('does not support checksum negotiation'),
}),
})
);
const writable = provider.createAssetsWriteStream();
if (writable instanceof Promise) {
throw new Error('Expected synchronous Writable');
}
const asset: IAsset = {
filename: 'f.bin',
filepath: '/tmp/f.bin',
stats: { size: 3 } as IAsset['stats'],
metadata: { id: 1 },
stream: Readable.from([Buffer.from([1, 2, 3])]),
};
await new Promise<void>((resolve, reject) => {
writable.write(asset, (err) => (err ? reject(err) : resolve()));
});
await new Promise<void>((resolve, reject) => {
writable.end((err) => (err ? reject(err) : resolve()));
});
const endItem = streamBatches.flat().find((i) => (i as { action: string }).action === 'end') as
| { checksum?: unknown }
| undefined;
expect(endItem?.checksum).toBeUndefined();
});
});
@@ -0,0 +1,198 @@
import { createHash } from 'crypto';
import { Readable } from 'stream';
import type { IAsset } from '../../../../../types';
import { createRemoteStrapiDestinationProvider } from '..';
import type { IRemoteStrapiDestinationProviderOptions } from '..';
const defaultOptions: IRemoteStrapiDestinationProviderOptions = {
strategy: 'restore',
url: new URL('http://localhost:1337/admin'),
auth: undefined,
};
function mockPushDispatcher() {
const streamBatches: unknown[][] = [];
return {
streamBatches,
dispatcher: {
dispatchTransferStep: jest.fn(
async (msg: { action: string; step?: string; data?: unknown }) => {
if (msg.action === 'stream' && msg.step === 'assets' && Array.isArray(msg.data)) {
streamBatches.push(msg.data);
}
if (msg.action === 'start') {
return { ok: true };
}
if (msg.action === 'end') {
return { ok: true, stats: { started: 1, finished: 1 } };
}
return null;
}
),
dispatchCommand: jest.fn(),
dispatchTransferAction: jest.fn(),
setTransferProperties: jest.fn(),
dispatch: jest.fn(),
transferID: 't1',
transferKind: 'push' as const,
},
};
}
const writeOneAsset = async (writable: NodeJS.WritableStream, asset: IAsset): Promise<void> => {
await new Promise<void>((resolve, reject) => {
writable.write(asset, (err: Error | null | undefined) => (err ? reject(err) : resolve()));
});
await new Promise<void>((resolve, reject) => {
writable.end((err: Error | null | undefined) => (err ? reject(err) : resolve()));
});
};
describe('Remote Strapi destination provider — push assets write stream', () => {
test('flushes before asset completes when decoded payload exceeds 1MiB and uses base64 stream chunks', async () => {
const { streamBatches, dispatcher } = mockPushDispatcher();
const provider = createRemoteStrapiDestinationProvider(defaultOptions);
provider.dispatcher = dispatcher as unknown as typeof provider.dispatcher;
const writable = provider.createAssetsWriteStream();
if (writable instanceof Promise) {
throw new Error('Expected synchronous Writable');
}
const chunkSize = 200_000;
const numChunks = 8;
const stream = Readable.from(
Array.from({ length: numChunks }, () => Buffer.alloc(chunkSize, 0xab))
);
const asset: IAsset = {
filename: 'big.bin',
filepath: '/tmp/big.bin',
stats: { size: chunkSize * numChunks } as IAsset['stats'],
metadata: { id: 1 },
stream,
};
await new Promise<void>((resolve, reject) => {
writable.write(asset, (err) => (err ? reject(err) : resolve()));
});
await new Promise<void>((resolve, reject) => {
writable.end((err) => (err ? reject(err) : resolve()));
});
expect(streamBatches.length).toBeGreaterThanOrEqual(2);
const streamItems = streamBatches
.flat()
.filter((i) => (i as { action: string }).action === 'stream');
expect(streamItems).toHaveLength(numChunks);
for (const item of streamItems) {
const row = item as { action: string; encoding?: string; data: unknown };
expect(row.encoding).toBe('base64');
expect(typeof row.data).toBe('string');
}
}, 30_000);
test('pushes many small assets with one start/end per asset (batches flushed, not accumulated)', async () => {
const { streamBatches, dispatcher } = mockPushDispatcher();
const provider = createRemoteStrapiDestinationProvider(defaultOptions);
provider.dispatcher = dispatcher as unknown as typeof provider.dispatcher;
const writable = provider.createAssetsWriteStream();
if (writable instanceof Promise) {
throw new Error('Expected synchronous Writable');
}
const assetCount = 30;
for (let i = 0; i < assetCount; i += 1) {
const stream = Readable.from([Buffer.from([i % 256, 0x02, 0x03])]);
const asset: IAsset = {
filename: `f-${i}.bin`,
filepath: `/tmp/f-${i}.bin`,
stats: { size: 3 } as IAsset['stats'],
metadata: { id: i },
stream,
};
await new Promise<void>((resolve, reject) => {
writable.write(asset, (err) => (err ? reject(err) : resolve()));
});
}
await new Promise<void>((resolve, reject) => {
writable.end((err) => (err ? reject(err) : resolve()));
});
const flat = streamBatches.flat() as { action: string }[];
expect(flat.filter((i) => i.action === 'start')).toHaveLength(assetCount);
expect(flat.filter((i) => i.action === 'end')).toHaveLength(assetCount);
expect(streamBatches.length).toBeGreaterThan(0);
expect(dispatcher.dispatchTransferStep).toHaveBeenCalled();
}, 30_000);
test('includes per-asset checksum in end row when verifyChecksums is enabled', async () => {
const { streamBatches, dispatcher } = mockPushDispatcher();
const provider = createRemoteStrapiDestinationProvider({
...defaultOptions,
verifyChecksums: true,
});
provider.dispatcher = dispatcher as unknown as typeof provider.dispatcher;
const writable = provider.createAssetsWriteStream();
if (writable instanceof Promise) {
throw new Error('Expected synchronous Writable');
}
const payload = [Buffer.from('hello'), Buffer.from(' world')];
const stream = Readable.from(payload);
const asset: IAsset = {
filename: 'checksum.bin',
filepath: '/tmp/checksum.bin',
stats: { size: 11 } as IAsset['stats'],
metadata: { id: 1 },
stream,
};
await writeOneAsset(writable, asset);
const endItem = streamBatches.flat().find((i) => (i as { action: string }).action === 'end') as
| { checksum?: { algorithm: string; value: string } }
| undefined;
const expected = createHash('sha256').update(Buffer.concat(payload)).digest('hex');
expect(endItem?.checksum).toEqual({ algorithm: 'sha256', value: expected });
});
test('does not include checksum in end row when verifyChecksums is disabled', async () => {
const { streamBatches, dispatcher } = mockPushDispatcher();
const provider = createRemoteStrapiDestinationProvider({
...defaultOptions,
verifyChecksums: false,
});
provider.dispatcher = dispatcher as unknown as typeof provider.dispatcher;
const writable = provider.createAssetsWriteStream();
if (writable instanceof Promise) {
throw new Error('Expected synchronous Writable');
}
const stream = Readable.from([Buffer.from('abc')]);
const asset: IAsset = {
filename: 'no-checksum.bin',
filepath: '/tmp/no-checksum.bin',
stats: { size: 3 } as IAsset['stats'],
metadata: { id: 1 },
stream,
};
await writeOneAsset(writable, asset);
const endItem = streamBatches.flat().find((i) => (i as { action: string }).action === 'end') as
| { checksum?: { algorithm: string; value: string } }
| undefined;
expect(endItem?.checksum).toBeUndefined();
});
});
@@ -3,13 +3,22 @@ import { TRANSFER_PATH } from '../../../remote/constants';
import { CommandMessage } from '../../../../../types/remote/protocol/client';
import { createDispatcher } from '../../utils';
jest.useFakeTimers();
jest.mock('ws', () => ({
WebSocket: jest.fn().mockImplementation(() => {
let onMessage: ((data: string) => void) | undefined;
return {
...jest.requireActual('ws').WebSocket,
send: jest.fn(),
once: jest.fn(),
send: jest.fn((payload: string, cb?: (err?: Error) => void) => {
cb?.();
const parsed = JSON.parse(payload) as { uuid: string };
queueMicrotask(() => {
onMessage?.(JSON.stringify({ uuid: parsed.uuid, data: null }));
});
}),
once: jest.fn((event: string, handler: (data: string) => void) => {
if (event === 'message') {
onMessage = handler;
}
}),
};
}),
}));
@@ -19,14 +28,14 @@ afterEach(() => {
});
describe('Remote Strapi Destination Utils', () => {
test('Dispatch method sends payload', () => {
test('Dispatch method sends payload', async () => {
const ws = new WebSocket(`ws://test/admin${TRANSFER_PATH}`);
const message: CommandMessage = {
type: 'command',
command: 'status',
};
createDispatcher(ws).dispatch(message);
await createDispatcher(ws).dispatch(message);
expect.extend({
toContain(receivedString, expected) {
@@ -44,4 +53,25 @@ describe('Remote Strapi Destination Utils', () => {
// @ts-ignore
expect(ws.send).toHaveBeenCalledWith(expect.toContain(message), expect.anything());
});
test('dispatch stringifies typed arrays with websocket replacer', async () => {
const ws = new WebSocket(`ws://test/admin${TRANSFER_PATH}`);
const message = {
type: 'transfer' as const,
kind: 'step' as const,
step: 'entities' as const,
action: 'stream' as const,
data: [{ bytes: new Uint8Array([0xde, 0xad]) }],
};
await createDispatcher(ws).dispatch(message);
// @ts-ignore
const payload = (ws.send as jest.Mock).mock.calls[0]?.[0];
const parsed = JSON.parse(payload) as {
data: Array<{ bytes: string }>;
};
expect(parsed.data[0].bytes).toBe(Buffer.from([0xde, 0xad]).toString('base64'));
});
});
@@ -1,4 +1,4 @@
import { randomUUID } from 'crypto';
import { createHash, randomUUID } from 'crypto';
import { Writable } from 'stream';
import { WebSocket } from 'ws';
import { once } from 'lodash/fp';
@@ -19,6 +19,10 @@ import type { Client, Server, Auth } from '../../../../types/remote/protocol';
import type { ILocalStrapiDestinationProviderOptions } from '../local-destination';
import { TRANSFER_PATH } from '../../remote/constants';
import { ProviderTransferError, ProviderValidationError } from '../../../errors/providers';
import {
createTransferAssetStreamChunk,
transferAssetStreamChunkByteLength,
} from '../../../utils/transfer-asset-chunk';
export interface IRemoteStrapiDestinationProviderOptions
extends Pick<ILocalStrapiDestinationProviderOptions, 'restore' | 'strategy'> {
@@ -28,6 +32,8 @@ export interface IRemoteStrapiDestinationProviderOptions
retryMessageTimeout: number; // milliseconds to wait for a response from a message
retryMessageMaxRetries: number; // max number of retries for a message before aborting transfer
};
/** Include per-asset stream checksums and require peers to validate on receive. */
verifyChecksums?: boolean;
}
const jsonLength = (obj: object) => Buffer.byteLength(JSON.stringify(obj));
@@ -49,11 +55,14 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
#diagnostics?: IDiagnosticReporter;
#checksumsEnabled = false;
constructor(options: IRemoteStrapiDestinationProviderOptions) {
this.options = options;
this.ws = null;
this.dispatcher = null;
this.transferID = null;
this.#checksumsEnabled = options.verifyChecksums === true;
this.resetStats();
}
@@ -69,16 +78,29 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
async initTransfer(): Promise<string> {
const { strategy, restore } = this.options;
const wantsChecksums = this.options.verifyChecksums === true;
const query = this.dispatcher?.dispatchCommand({
command: 'init',
params: { options: { strategy, restore }, transfer: 'push' },
params: {
options: { strategy, restore },
transfer: 'push',
...(wantsChecksums ? { checksums: true } : {}),
},
});
const res = (await query) as Server.Payload<Server.InitMessage>;
const res = (await query) as
| (Server.Payload<Server.InitMessage> & { checksums?: boolean })
| null;
if (!res?.transferID) {
throw new ProviderTransferError('Init failed, invalid response from the server');
}
this.#checksumsEnabled = wantsChecksums && res.checksums === true;
if (wantsChecksums && res.checksums !== true) {
this.#reportWarning(
'[Data transfer][push] Checksums were requested but the remote does not support checksum negotiation; continuing without checksum validation.'
);
}
this.resetStats();
@@ -233,6 +255,17 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
});
}
#reportWarning(message: string) {
this.#diagnostics?.report({
details: {
createdAt: new Date(),
message,
origin: 'remote-destination-provider',
},
kind: 'warning',
});
}
async bootstrap(diagnostics?: IDiagnosticReporter): Promise<void> {
this.#diagnostics = diagnostics;
const { url, auth } = this.options;
@@ -354,13 +387,11 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
createAssetsWriteStream(): Writable | Promise<Writable> {
let batch: Client.TransferAssetFlow[] = [];
let hasStarted = false;
const verifyChecksums = this.#checksumsEnabled;
const batchSize = 1024 * 1024; // 1MB;
const batchLength = () => {
return batch.reduce(
(acc, chunk) => (chunk.action === 'stream' ? acc + chunk.data.byteLength : acc),
0
);
return batch.reduce((acc, chunk) => acc + transferAssetStreamChunkByteLength(chunk), 0);
};
const startAssetsTransferOnce = this.#startStepOnce('assets');
@@ -409,6 +440,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
const assetID = randomUUID();
const { filename, filepath, stats, stream, metadata } = asset;
const checksumHash = verifyChecksums ? createHash('sha256') : undefined;
try {
await safePush({
@@ -418,16 +450,21 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
});
for await (const chunk of stream) {
await safePush({ action: 'stream', assetID, data: chunk });
checksumHash?.update(chunk);
await safePush(createTransferAssetStreamChunk(assetID, chunk));
}
await safePush({ action: 'end', assetID });
await safePush({
action: 'end',
assetID,
...(checksumHash
? { checksum: { algorithm: 'sha256' as const, value: checksumHash.digest('hex') } }
: {}),
});
callback();
} catch (error) {
if (error instanceof Error) {
callback(error);
}
callback(error instanceof Error ? error : new Error(String(error)));
}
},
});
@@ -0,0 +1,132 @@
import { EventEmitter } from 'events';
import { pipeline } from 'stream/promises';
import { Writable } from 'stream';
import type { Core } from '@strapi/types';
import type { IAsset } from '../../../../../types';
import { createRemoteStrapiSourceProvider } from '..';
import { connectToWebsocket, createDispatcher } from '../../utils';
jest.mock('../../utils', () => ({
...jest.requireActual('../../utils'),
connectToWebsocket: jest.fn(),
createDispatcher: jest.fn(),
}));
class MockWebSocket extends EventEmitter {
send = jest.fn((_payload: string, cb?: (err?: Error) => void) => cb?.());
close = jest.fn();
}
const transferMessageBuffer = (opts: {
uuid: string;
processId: string;
ended?: boolean;
error?: unknown;
data?: unknown;
}) =>
Buffer.from(
JSON.stringify({
uuid: opts.uuid,
data: {
type: 'transfer',
id: opts.processId,
ended: opts.ended ?? false,
error: opts.error ?? null,
data: opts.data ?? null,
},
})
);
const oneAssetBatch = (assetID: string) => [
{
action: 'start' as const,
assetID,
data: {
filename: 't.bin',
metadata: { id: 1 },
},
},
{
action: 'stream' as const,
assetID,
data: { type: 'Buffer' as const, data: [1, 2, 3] },
},
{ action: 'end' as const, assetID },
];
describe('Remote source checksum negotiation', () => {
test('warns and continues without checksums when peer does not support checksum negotiation', async () => {
const ws = new MockWebSocket();
(connectToWebsocket as jest.Mock).mockResolvedValue(ws);
const dispatcher = {
dispatchCommand: jest.fn().mockResolvedValue({ transferID: 't1' }), // no `checksums` support
dispatchTransferAction: jest.fn().mockResolvedValue(null),
dispatchTransferStep: jest.fn(async (msg: { action: string }) => {
if (msg.action === 'start') {
return { id: 'pid-1' };
}
return null;
}),
setTransferProperties: jest.fn(),
dispatch: jest.fn(),
transferID: 't1',
transferKind: 'pull' as const,
};
(createDispatcher as jest.Mock).mockReturnValue(dispatcher);
const diagnostics = { report: jest.fn() };
const provider = createRemoteStrapiSourceProvider({
url: new URL('http://localhost:1337/admin'),
getStrapi: () => ({}) as Core.Strapi,
verifyChecksums: true,
});
await provider.bootstrap(diagnostics as any);
expect(diagnostics.report).toHaveBeenCalledWith(
expect.objectContaining({
kind: 'warning',
details: expect.objectContaining({
message: expect.stringContaining('does not support checksum negotiation'),
}),
})
);
const pass = await provider.createAssetsReadStream();
const drainPass = pipeline(
pass,
new Writable({
objectMode: true,
write(asset: IAsset, _enc, cb) {
pipeline(
asset.stream,
new Writable({
write(_chunk, _e, c) {
c();
},
})
).then(
() => cb(),
(err) => cb(err)
);
},
})
);
ws.emit(
'message',
transferMessageBuffer({
uuid: 'u-1',
processId: 'pid-1',
data: [oneAssetBatch('a1')],
})
);
ws.emit('message', transferMessageBuffer({ uuid: 'u-end', processId: 'pid-1', ended: true }));
await expect(drainPass).resolves.toBeUndefined();
});
});
@@ -0,0 +1,506 @@
import { EventEmitter } from 'node:events';
import { createHash } from 'crypto';
import { pipeline } from 'stream/promises';
import { Writable } from 'stream';
import type { Core } from '@strapi/types';
import type { WebSocket } from 'ws';
import type { IAsset } from '../../../../../types';
import { createRemoteStrapiSourceProvider } from '..';
import { ProviderTransferError } from '../../../../errors/providers';
/** WebSocket frame shape expected by RemoteStrapiSourceProvider.#createStageReadStream */
function transferMessageBuffer(opts: {
uuid: string;
processId: string;
ended?: boolean;
error?: unknown;
data?: unknown;
}) {
return Buffer.from(
JSON.stringify({
uuid: opts.uuid,
data: {
type: 'transfer',
id: opts.processId,
ended: opts.ended ?? false,
error: opts.error ?? null,
data: opts.data ?? null,
},
})
);
}
function oneAssetBatch(
assetID: string,
checksum?: {
algorithm: 'sha256';
value: string;
}
) {
const chunk = Buffer.from([0x01, 0x02, 0x03]);
return [
{
action: 'start' as const,
assetID,
data: {
filename: 't.bin',
metadata: { id: 1 },
},
},
{
action: 'stream' as const,
assetID,
data: { type: 'Buffer' as const, data: Array.from(chunk) },
},
{ action: 'end' as const, assetID, ...(checksum ? { checksum } : {}) },
];
}
/**
* Minimal WebSocket mock: provider only needs `once('message')` and `send` for ACKs.
*/
class MockTransferWebSocket extends EventEmitter {
send = jest.fn((_payload: string, cb?: (err?: Error) => void) => {
cb?.();
});
}
const createProviderWithMockDispatcher = (
ws: MockTransferWebSocket,
processId: string,
options: { verifyChecksums?: boolean } = {}
) => {
const provider = createRemoteStrapiSourceProvider({
url: new URL('http://localhost:1337/admin'),
getStrapi: () => ({}) as Core.Strapi,
streamTimeout: 60_000,
...(options.verifyChecksums !== undefined ? { verifyChecksums: options.verifyChecksums } : {}),
});
provider.ws = ws as unknown as WebSocket;
provider.dispatcher = {
dispatchTransferStep: jest.fn(async (msg: { action: string }) => {
if (msg.action === 'start') {
return { id: processId };
}
return null;
}),
setTransferProperties: jest.fn(),
dispatchCommand: jest.fn(),
dispatch: jest.fn(),
dispatchTransferAction: jest.fn(),
transferID: 't1',
transferKind: 'pull',
} as unknown as typeof provider.dispatcher;
return provider;
};
describe('Remote Strapi source provider — pull assets stream', () => {
const processId = 'flush-process-id';
const flushPromises = () =>
new Promise<void>((resolve) => {
setImmediate(() => {
resolve();
});
});
test('after each asset completes the same assetID can start again (registry must not retain closed assets)', async () => {
const ws = new MockTransferWebSocket();
const dispatchTransferStep = jest.fn(async (msg: { action: string; step?: string }) => {
if (msg.action === 'start') {
return { id: processId };
}
return null;
});
const provider = createRemoteStrapiSourceProvider({
url: new URL('http://localhost:1337/admin'),
getStrapi: () => ({}) as Core.Strapi,
streamTimeout: 60_000,
});
provider.ws = ws as unknown as WebSocket;
provider.dispatcher = {
dispatchTransferStep,
setTransferProperties: jest.fn(),
dispatchCommand: jest.fn(),
dispatch: jest.fn(),
dispatchTransferAction: jest.fn(),
transferID: 't1',
transferKind: 'pull',
} as unknown as typeof provider.dispatcher;
const pass = await provider.createAssetsReadStream();
const reuseId = 'reused-asset-id';
const cycles = 40;
const drainPass = pipeline(
pass,
new Writable({
objectMode: true,
highWaterMark: 1,
write(asset: IAsset, _enc, cb) {
const rs = asset.stream;
if (!rs) {
cb();
return;
}
pipeline(
rs,
new Writable({
write(_chunk, _e, c) {
c();
},
})
).then(
() => {
cb();
},
(err) => {
cb(err);
}
);
},
})
);
for (let i = 0; i < cycles; i += 1) {
ws.emit(
'message',
transferMessageBuffer({
uuid: `u-${i}`,
processId,
data: [oneAssetBatch(reuseId)],
})
);
await flushPromises();
}
ws.emit(
'message',
transferMessageBuffer({
uuid: 'u-end',
processId,
ended: true,
})
);
await drainPass;
// Client start + end for the assets step, and one WebSocket ACK per incoming message (cycles + end).
expect(dispatchTransferStep).toHaveBeenCalledWith(
expect.objectContaining({ action: 'start', step: 'assets' }),
expect.objectContaining({
retryOverrides: expect.objectContaining({
retryMessageTimeout: 120_000,
retryMessageMaxRetries: 30,
}),
})
);
expect(dispatchTransferStep).toHaveBeenCalledWith(
expect.objectContaining({ action: 'end', step: 'assets' })
);
expect(dispatchTransferStep).toHaveBeenCalledTimes(2);
expect(ws.send).toHaveBeenCalledTimes(cycles + 1);
}, 15_000);
test('many small asset batches complete without error (registry does not grow with batch count)', async () => {
const ws = new MockTransferWebSocket();
const dispatchTransferStep = jest.fn(async (msg: { action: string }) => {
if (msg.action === 'start') {
return { id: processId };
}
return null;
});
const provider = createRemoteStrapiSourceProvider({
url: new URL('http://localhost:1337/admin'),
getStrapi: () => ({}) as Core.Strapi,
streamTimeout: 60_000,
});
provider.ws = ws as unknown as WebSocket;
provider.dispatcher = {
dispatchTransferStep,
setTransferProperties: jest.fn(),
dispatchCommand: jest.fn(),
dispatch: jest.fn(),
dispatchTransferAction: jest.fn(),
transferID: 't1',
transferKind: 'pull',
} as unknown as typeof provider.dispatcher;
const pass = await provider.createAssetsReadStream();
const batchCount = 25;
const drainPass = pipeline(
pass,
new Writable({
objectMode: true,
highWaterMark: 1,
write(asset: IAsset, _enc, cb) {
const rs = asset.stream;
if (!rs) {
cb();
return;
}
pipeline(
rs,
new Writable({
write(_chunk, _e, c) {
c();
},
})
).then(
() => {
cb();
},
(err) => {
cb(err);
}
);
},
})
);
for (let i = 0; i < batchCount; i += 1) {
const id = `unique-${i}`;
ws.emit(
'message',
transferMessageBuffer({
uuid: `batch-${i}`,
processId,
data: [oneAssetBatch(id)],
})
);
await flushPromises();
}
ws.emit(
'message',
transferMessageBuffer({
uuid: 'u-end',
processId,
ended: true,
})
);
await drainPass;
expect(dispatchTransferStep).toHaveBeenCalledWith(
expect.objectContaining({ action: 'start', step: 'assets' }),
expect.objectContaining({
retryOverrides: expect.objectContaining({
retryMessageTimeout: 120_000,
retryMessageMaxRetries: 30,
}),
})
);
expect(dispatchTransferStep).toHaveBeenCalledWith(
expect.objectContaining({ action: 'end', step: 'assets' })
);
expect(dispatchTransferStep).toHaveBeenCalledTimes(2);
expect(ws.send).toHaveBeenCalledTimes(batchCount + 1);
}, 20_000);
test('throws on checksum mismatch when verifyChecksums is enabled', async () => {
const ws = new MockTransferWebSocket();
const provider = createProviderWithMockDispatcher(ws, processId, { verifyChecksums: true });
const pass = await provider.createAssetsReadStream();
const id = 'asset-with-bad-checksum';
const badChecksum = createHash('sha256').update(Buffer.from('wrong')).digest('hex');
const drainPass = pipeline(
pass,
new Writable({
objectMode: true,
write(asset: IAsset, _enc, cb) {
pipeline(
asset.stream,
new Writable({
write(_chunk, _e, c) {
c();
},
})
).then(
() => cb(),
(err) => cb(err)
);
},
})
);
ws.emit(
'message',
transferMessageBuffer({
uuid: 'checksum-1',
processId,
data: [
oneAssetBatch(id, {
algorithm: 'sha256',
value: badChecksum,
}),
],
})
);
await flushPromises();
ws.emit(
'message',
transferMessageBuffer({
uuid: 'checksum-end',
processId,
ended: true,
})
);
const checksumErr = await drainPass.catch((e: unknown) => e);
expect(checksumErr).toBeInstanceOf(ProviderTransferError);
expect((checksumErr as Error).message).toMatch(/Checksum mismatch/);
});
test('throws ProviderTransferError when checksum is required but missing', async () => {
const ws = new MockTransferWebSocket();
const provider = createProviderWithMockDispatcher(ws, processId, { verifyChecksums: true });
const pass = await provider.createAssetsReadStream();
const id = 'asset-missing-checksum';
const drainPass = pipeline(
pass,
new Writable({
objectMode: true,
write(asset: IAsset, _enc, cb) {
pipeline(
asset.stream,
new Writable({
write(_chunk, _e, c) {
c();
},
})
).then(
() => cb(),
(err) => cb(err)
);
},
})
);
ws.emit(
'message',
transferMessageBuffer({
uuid: 'missing-1',
processId,
data: [oneAssetBatch(id)],
})
);
await flushPromises();
ws.emit(
'message',
transferMessageBuffer({
uuid: 'missing-end',
processId,
ended: true,
})
);
const missingErr = await drainPass.catch((e: unknown) => e);
expect(missingErr).toBeInstanceOf(ProviderTransferError);
expect((missingErr as Error).message).toMatch(/missing checksum/i);
});
test('accepts matching checksum when verifyChecksums is enabled', async () => {
const ws = new MockTransferWebSocket();
const provider = createProviderWithMockDispatcher(ws, processId, { verifyChecksums: true });
const pass = await provider.createAssetsReadStream();
const id = 'asset-with-good-checksum';
const goodChecksum = createHash('sha256')
.update(Buffer.from([0x01, 0x02, 0x03]))
.digest('hex');
const drainPass = pipeline(
pass,
new Writable({
objectMode: true,
write(asset: IAsset, _enc, cb) {
pipeline(
asset.stream,
new Writable({
write(_chunk, _e, c) {
c();
},
})
).then(
() => cb(),
(err) => cb(err)
);
},
})
);
ws.emit(
'message',
transferMessageBuffer({
uuid: 'good-1',
processId,
data: [
oneAssetBatch(id, {
algorithm: 'sha256',
value: goodChecksum,
}),
],
})
);
await flushPromises();
ws.emit('message', transferMessageBuffer({ uuid: 'good-end', processId, ended: true }));
await expect(drainPass).resolves.toBeUndefined();
});
test('ignores missing checksum when verifyChecksums is disabled', async () => {
const ws = new MockTransferWebSocket();
const provider = createProviderWithMockDispatcher(ws, processId, { verifyChecksums: false });
const pass = await provider.createAssetsReadStream();
const id = 'asset-without-checksum-disabled';
const drainPass = pipeline(
pass,
new Writable({
objectMode: true,
write(asset: IAsset, _enc, cb) {
pipeline(
asset.stream,
new Writable({
write(_chunk, _e, c) {
c();
},
})
).then(
() => cb(),
(err) => cb(err)
);
},
})
);
ws.emit(
'message',
transferMessageBuffer({
uuid: 'disabled-1',
processId,
data: [oneAssetBatch(id)],
})
);
await flushPromises();
ws.emit('message', transferMessageBuffer({ uuid: 'disabled-end', processId, ended: true }));
await expect(drainPass).resolves.toBeUndefined();
});
});
@@ -1,3 +1,4 @@
import { createHash, type Hash } from 'crypto';
import { PassThrough, Readable, Writable } from 'stream';
import type { Struct, Utils } from '@strapi/types';
import { WebSocket } from 'ws';
@@ -11,14 +12,32 @@ import type {
MaybePromise,
Protocol,
ProviderType,
StageTotalsEstimate,
TransferStage,
} from '../../../../types';
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
import { Client, Server, Auth } from '../../../../types/remote/protocol';
import { ProviderTransferError, ProviderValidationError } from '../../../errors/providers';
import { TRANSFER_PATH } from '../../remote/constants';
import { decodeTransferAssetStreamItem } from '../../../utils/transfer-asset-chunk';
import { write } from '../../../utils/writable-async-write';
import { ILocalStrapiSourceProviderOptions } from '../local-source';
import { createDispatcher, connectToWebsocket, trimTrailingSlash } from '../utils';
import {
createDispatcher,
connectToWebsocket,
trimTrailingSlash,
type RetryMessageOptions,
} from '../utils';
/**
* Pull server answers `assets` step `start` only after `estimateAssetTotals` (DB stream; remote sizes from DB when complete, else HTTP like `createAssetsStream`).
* That can exceed the default dispatcher wait (~30s between resends, a few minutes total). This message
* uses a longer window so large libraries do not fail with `Request timed out` before totals are returned.
*/
const ASSETS_START_RETRY_OVERRIDES: Partial<RetryMessageOptions> = {
retryMessageTimeout: 120_000,
retryMessageMaxRetries: 30,
};
export interface IRemoteStrapiSourceProviderOptions extends ILocalStrapiSourceProviderOptions {
url: URL; // the url of the remote Strapi admin
@@ -27,7 +46,10 @@ export interface IRemoteStrapiSourceProviderOptions extends ILocalStrapiSourcePr
retryMessageTimeout: number; // milliseconds to wait for a response from a message
retryMessageMaxRetries: number; // max number of retries for a message before aborting transfer
};
streamTimeout?: number; // milliseconds to wait between chunks of an asset before aborting the transfer
/** Max ms without forward progress on an asset (new remote chunk accepted or chunk fully handed to the asset stream). */
streamTimeout?: number;
/** Require per-asset checksum verification for transferred asset bytes. */
verifyChecksums?: boolean;
}
type QueueableAction = Protocol.Client.TransferAssetFlow &
@@ -45,7 +67,8 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
dispatcher: ReturnType<typeof createDispatcher> | null;
defaultOptions: Partial<IRemoteStrapiSourceProviderOptions> = {
streamTimeout: 15000,
// Large files + JSON/WS backpressure can go minutes between *messages* while bytes still drain locally
streamTimeout: 300_000,
};
constructor(options: IRemoteStrapiSourceProviderOptions) {
@@ -53,6 +76,7 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
...this.defaultOptions,
...options,
};
this.#checksumsEnabled = this.options.verifyChecksums === true;
this.ws = null;
this.dispatcher = null;
@@ -62,15 +86,36 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
#diagnostics?: IDiagnosticReporter;
#pullAssetStreamWireSampleLogged = false;
#checksumsEnabled = false;
/** Set from pull server `start` response for `assets` when present (for engine `getStageTotals`). */
#cachedAssetsTotals?: StageTotalsEstimate;
async #createStageReadStream(stage: Exclude<TransferStage, 'schemas'>) {
if (stage === 'assets') {
this.#cachedAssetsTotals = undefined;
}
const startResult = await this.#startStep(stage);
if (startResult instanceof Error) {
throw startResult;
}
const { id: processID } = startResult as { id: string };
const { id: processID, totals } = startResult as {
id: string;
totals?: StageTotalsEstimate;
};
if (stage === 'assets' && totals && (totals.totalBytes != null || totals.totalCount != null)) {
this.#cachedAssetsTotals = totals;
}
// Default object-mode HWM (~16 chunks). Do not await `drain` on manual `push` while `pipe()`
// is attached — drain/`readableLength` races reliably deadlock after a few 1MiB asset frames.
// Backpressure for pull assets is enforced by the Writable below (`highWaterMark: 1`).
const stream = new PassThrough({ objectMode: true });
const listener = async (raw: Buffer) => {
@@ -98,9 +143,8 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
return;
}
// if we get a single items instead of a batch
for (const item of castArray(data)) {
stream.push(item);
stream.push(item as Parameters<PassThrough['push']>[0]);
}
this.ws?.once('message', listener);
@@ -121,18 +165,6 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
return this.#createStageReadStream('links');
}
writeAsync = <T>(stream: Writable, data: T) => {
return new Promise<void>((resolve, reject) => {
stream.write(data, (error) => {
if (error) {
reject(error);
}
resolve();
});
});
};
async createAssetsReadStream(): Promise<Readable> {
// Create the streams used to transfer the assets
const stream = await this.#createStageReadStream('assets');
@@ -146,93 +178,150 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
queue: Array<QueueableAction>;
status: 'ok' | 'closed' | 'errored';
timeout?: NodeJS.Timeout;
checksumHash?: Hash;
};
} = {};
// Watch for stalled assets; if we don't receive a chunk within timeout, abort transfer
// Watch for stalled assets: no remote chunks and no completed writes to the asset stream for streamTimeout ms
const resetTimeout = (assetID: string) => {
if (!assets[assetID]) {
return;
}
if (assets[assetID].timeout) {
clearTimeout(assets[assetID].timeout);
}
assets[assetID].timeout = setTimeout(() => {
if (!assets[assetID]) {
return;
}
this.#reportInfo(`Asset ${assetID} transfer stalled, aborting.`);
assets[assetID].status = 'errored';
assets[assetID].stream.destroy(new Error(`Asset ${assetID} transfer timed out`));
}, this.options.streamTimeout);
};
stream
/**
* Process a payload of many transfer assets and performs the following tasks:
* - Start: creates a stream for new assets.
* - Stream: writes asset chunks to the asset's stream.
* - End: closes the stream after the asset s transferred and cleanup related resources.
*/
.on('data', async (payload: Protocol.Client.TransferAssetFlow[]) => {
for (const item of payload) {
const { action, assetID } = item;
const clearStallTimeoutForAsset = (assetID: string) => {
const entry = assets[assetID];
if (entry?.timeout) {
clearTimeout(entry.timeout);
entry.timeout = undefined;
}
};
// Creates the stream to send the incoming asset through
if (action === 'start') {
// if a transfer has already been started for the same asset ID, something is wrong
if (assets[assetID]) {
throw new Error(`Asset ${assetID} already started`);
const clearAllStallTimeouts = () => {
for (const id of Object.keys(assets)) {
clearStallTimeoutForAsset(id);
}
};
/**
* Serialize asset batch handling: `Readable.on('data', async …)` does not apply backpressure,
* so we pipe through a Writable with highWaterMark 1 so only one batch is in flight.
*/
const processAssetPayload = async (payload: Protocol.Client.TransferAssetFlow[]) => {
for (const item of payload) {
const { action, assetID } = item;
if (action === 'start') {
if (assets[assetID]) {
throw new Error(`Asset ${assetID} already started`);
}
this.#reportInfo(`Asset ${assetID} starting`);
assets[assetID] = {
...item.data,
stream: new PassThrough(),
status: 'ok',
queue: [],
...(this.#checksumsEnabled ? { checksumHash: createHash('sha256') } : {}),
};
resetTimeout(assetID);
await write(pass, assets[assetID]);
} else if (action === 'stream' || action === 'end') {
if (!assets[assetID]) {
throw new Error(`No id matching ${assetID} for stream action`);
}
if (action === 'stream') {
if (!this.#pullAssetStreamWireSampleLogged) {
this.#pullAssetStreamWireSampleLogged = true;
const { data } = item;
// Same legacy shape `decodeTransferAssetStreamData` accepts after JSON.parse (proof, not frame-size guess).
const legacyBufferJson =
data &&
typeof data === 'object' &&
!Buffer.isBuffer(data) &&
(data as { type?: string }).type === 'Buffer' &&
(Array.isArray((data as { data?: unknown }).data) ||
ArrayBuffer.isView((data as { data?: unknown }).data));
if (legacyBufferJson) {
this.#reportWarning(
'[Data transfer][pull] Remote is using legacy Buffer JSON for asset chunks (each byte as a JSON number). That uses much more memory during JSON.parse than base64. Upgrade the remote Strapi to a version that sends base64 asset chunks, or out-of-memory errors may still happen on large files.'
);
}
}
this.#reportInfo(`Asset ${assetID} starting`);
// Register the asset
assets[assetID] = {
...item.data,
stream: new PassThrough(),
status: 'ok',
queue: [],
};
resetTimeout(assetID);
// Connect the individual asset stream to the main asset stage stream
// Note: nothing is transferred until data chunks are fed to the asset stream
await this.writeAsync(pass, assets[assetID]);
} else {
clearTimeout(assets[assetID].timeout);
}
// Writes the asset's data chunks to their corresponding stream
// "end" is considered a chunk, but it's not a data chunk, it's a control message
// That is done so that we don't complicate the already complicated async processing of the queue
else if (action === 'stream' || action === 'end') {
// If the asset hasn't been registered, or if it's been closed already, something is wrong
if (!assets[assetID]) {
throw new Error(`No id matching ${assetID} for stream action`);
}
if (assets[assetID].status === 'closed') {
throw new Error(`Asset ${assetID} is closed`);
}
// On every action, reset the timeout timer
if (action === 'stream') {
resetTimeout(assetID);
} else {
clearTimeout(assets[assetID].timeout);
}
assets[assetID].queue.push(item);
}
}
if (assets[assetID].status === 'closed') {
throw new Error(`Asset ${assetID} is closed`);
}
assets[assetID].queue.push(item);
for (const assetID in assets) {
if (Object.prototype.hasOwnProperty.call(assets, assetID)) {
const asset = assets[assetID];
if (asset.queue?.length > 0) {
await processQueue(assetID);
}
}
}
};
// each new payload will start new processQueue calls, which may cause some extra calls
// it's essentially saying "start processing this asset again, I added more data to the queue"
for (const assetID in assets) {
if (Object.prototype.hasOwnProperty.call(assets, assetID)) {
const asset = assets[assetID];
if (asset.queue?.length > 0) {
await processQueue(assetID);
}
const processor = new Writable({
objectMode: true,
highWaterMark: 1,
write(payload: Protocol.Client.TransferAssetFlow[], _encoding, callback) {
processAssetPayload(payload).then(
() => {
callback();
},
(err: Error) => {
clearAllStallTimeouts();
stream.destroy(err);
callback(err);
}
}
})
.on('close', () => {
);
},
final(callback) {
pass.end();
});
callback();
},
});
processor.on('error', (err) => {
clearAllStallTimeouts();
pass.destroy(err);
});
stream.on('error', (err) => {
clearAllStallTimeouts();
processor.destroy(err);
pass.destroy(err);
});
stream.once('end', () => {
clearAllStallTimeouts();
});
stream.pipe(processor);
/**
* Start processing the queue for a given assetID
@@ -265,16 +354,21 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
// if this is an end chunk, close the asset stream
if (data.action === 'end') {
this.#reportInfo(`Ending asset stream for ${id}`);
await closeAssetStream(id);
await closeAssetStream(id, data.checksum);
break; // Exit the loop after closing the stream
}
// Save the current chunk
await writeChunkToStream(id, data);
} catch {
} catch (error) {
if (!assets[id]) {
throw new Error(`No id matching ${id} for writeAssetChunk`);
}
clearStallTimeoutForAsset(id);
if (error instanceof Error) {
throw error;
}
throw new Error(`Unexpected error while processing asset chunk for "${id}"`);
}
}
};
@@ -284,17 +378,23 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
*
* Only check if the targeted asset exists, no other validation is done.
*/
const writeChunkToStream = async (id: string, data: unknown) => {
const writeChunkToStream = async (id: string, item: QueueableAction) => {
const asset = assets[id];
if (!asset) {
throw new Error(`Failed to write asset chunk for "${id}". Asset not found.`);
}
const rawBuffer = data as { type: 'Buffer'; data: Uint8Array };
const chunk = Buffer.from(rawBuffer.data);
if (item.action !== 'stream') {
throw new Error(`Expected stream queue item for "${id}"`);
}
await this.writeAsync(asset.stream, chunk);
const chunk = decodeTransferAssetStreamItem(item);
asset.checksumHash?.update(chunk);
await write(asset.stream, chunk);
// Count slow draining as progress so backpressure on large chunks does not trip the stall timer
resetTimeout(id);
};
/**
@@ -302,22 +402,49 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
*
* It deletes the stream for the asset upon successful closure.
*/
const closeAssetStream = async (id: string) => {
const closeAssetStream = async (
id: string,
checksum?: { algorithm: 'sha256'; value: string }
) => {
if (!assets[id]) {
throw new Error(`Failed to close asset "${id}". Asset not found.`);
}
assets[id].status = 'closed';
const asset = assets[id];
// The queue processes stream chunks before `end`; the last `writeChunkToStream` calls
// `resetTimeout` after the `end` chunk already cleared the timer — clear again before closing.
clearStallTimeoutForAsset(id);
if (this.#checksumsEnabled) {
if (!checksum) {
throw new ProviderTransferError(
`Asset ${id} is missing checksum in transfer end payload`
);
}
if (checksum.algorithm !== 'sha256') {
throw new ProviderTransferError(
`Asset ${id} checksum algorithm "${checksum.algorithm}" is not supported`
);
}
const actual = asset.checksumHash?.digest('hex');
if (!actual || actual !== checksum.value) {
throw new ProviderTransferError(
`Checksum mismatch for asset "${id}" (expected ${checksum.value}, got ${actual ?? 'none'})`
);
}
}
asset.status = 'closed';
await new Promise<void>((resolve, reject) => {
const { stream } = assets[id];
const { stream } = asset;
stream
.on('close', () => {
delete assets[id];
resolve();
})
.on('error', (e) => {
assets[id].status = 'errored';
delete assets[id];
reject(new Error(`Failed to close asset "${id}". Asset stream error: ${e.toString()}`));
})
.end();
@@ -352,15 +479,25 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
}
async initTransfer(): Promise<string> {
const wantsChecksums = this.options.verifyChecksums === true;
const query = this.dispatcher?.dispatchCommand({
command: 'init',
...(wantsChecksums ? { params: { transfer: 'pull', checksums: true } } : {}),
});
const res = (await query) as Server.Payload<Server.InitMessage>;
const res = (await query) as
| (Server.Payload<Server.InitMessage> & { checksums?: boolean })
| null;
if (!res?.transferID) {
throw new ProviderTransferError('Init failed, invalid response from the server');
}
this.#checksumsEnabled = wantsChecksums && res.checksums === true;
if (wantsChecksums && res.checksums !== true) {
this.#reportWarning(
'[Data transfer][pull] Checksums were requested but the remote does not support checksum negotiation; continuing without checksum validation.'
);
}
return res.transferID;
}
@@ -376,6 +513,18 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
});
}
/** Reports a warning diagnostic (`kind: 'warning'`). Consumers (e.g. CLI) choose log levels and routing. */
#reportWarning(message: string) {
this.#diagnostics?.report({
details: {
createdAt: new Date(),
message,
origin: 'remote-source-provider',
},
kind: 'warning',
});
}
async bootstrap(diagnostics?: IDiagnosticReporter): Promise<void> {
this.#diagnostics = diagnostics;
const { url, auth } = this.options;
@@ -386,6 +535,8 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
url.pathname
)}${TRANSFER_PATH}/pull`;
this.#pullAssetStreamWireSampleLogged = false;
this.#reportInfo('establishing websocket connection');
// No auth defined, trying public access for transfer
if (!auth) {
@@ -416,7 +567,7 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
this.dispatcher = createDispatcher(this.ws, retryMessageOptions, (message: string) =>
this.#reportInfo(message)
);
this.#reportInfo('creating dispatcher');
this.#reportInfo('created dispatcher');
this.#reportInfo('initialize transfer');
const transferID = await this.initTransfer();
@@ -448,9 +599,20 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
return schemas ?? null;
}
async getStageTotals(stage: TransferStage): Promise<StageTotalsEstimate | null> {
if (stage !== 'assets') {
return null;
}
const cached = this.#cachedAssetsTotals;
return cached ?? null;
}
async #startStep<T extends Client.TransferPullStep>(step: T) {
try {
return await this.dispatcher?.dispatchTransferStep({ action: 'start', step });
return await this.dispatcher?.dispatchTransferStep(
{ action: 'start', step },
step === 'assets' ? { retryOverrides: ASSETS_START_RETRY_OVERRIDES } : undefined
);
} catch (e) {
if (e instanceof Error) {
return e;
@@ -11,20 +11,28 @@ import {
ProviderErrorDetails,
} from '../../errors/providers';
import { IDiagnosticReporter } from '../../utils/diagnostic';
import { stringifyTransferWebSocketPayload } from '../../utils/transfer-websocket-json';
interface IDispatcherState {
transfer?: { kind: Client.TransferKind; id: string };
}
export interface RetryMessageOptions {
retryMessageMaxRetries: number;
retryMessageTimeout: number;
}
interface IDispatchOptions {
attachTransfer?: boolean;
/** Merged onto the dispatcher's default `retryMessageOptions` for this message only. */
retryOverrides?: Partial<RetryMessageOptions>;
}
type Dispatch<T> = Omit<T, 'transferID' | 'uuid'>;
export const createDispatcher = (
ws: WebSocket,
retryMessageOptions = {
retryMessageOptions: RetryMessageOptions = {
retryMessageMaxRetries: 5,
retryMessageTimeout: 30000,
},
@@ -61,13 +69,18 @@ export const createDispatcher = (
`dispatching message action:${messageToSend.action} ${messageToSend.kind === 'step' ? `step:${messageToSend.step}` : ''} uuid:${uuid} sent:${numberOfTimesMessageWasSent}`
);
}
const stringifiedPayload = JSON.stringify(payload);
const stringifiedPayload = stringifyTransferWebSocketPayload(
payload as Record<string, unknown>
);
ws.send(stringifiedPayload, (error) => {
if (error) {
reject(error);
}
});
const { retryMessageMaxRetries, retryMessageTimeout } = retryMessageOptions;
const { retryMessageMaxRetries, retryMessageTimeout } = {
...retryMessageOptions,
...options.retryOverrides,
};
const sendPeriodically = () => {
if (numberOfTimesMessageWasSent <= retryMessageMaxRetries) {
numberOfTimesMessageWasSent += 1;
@@ -144,7 +157,8 @@ export const createDispatcher = (
payload: {
step: S;
action: A;
} & (A extends 'stream' ? { data: Client.GetTransferPushStreamData<S> } : unknown)
} & (A extends 'stream' ? { data: Client.GetTransferPushStreamData<S> } : unknown),
dispatchOptions?: Omit<IDispatchOptions, 'attachTransfer'>
) => {
const message: Dispatch<Client.TransferPushMessage> = {
type: 'transfer',
@@ -152,7 +166,9 @@ export const createDispatcher = (
...payload,
};
return dispatch<T>(message, { attachTransfer: true }) ?? Promise.resolve(null);
return (
dispatch<T>(message, { attachTransfer: true, ...dispatchOptions }) ?? Promise.resolve(null)
);
};
const setTransferProperties = (
@@ -1,12 +1,20 @@
import { Readable } from 'stream';
import { randomUUID } from 'crypto';
import { randomUUID, createHash } from 'crypto';
import type { Core } from '@strapi/types';
import { Handler } from './abstract';
import { handlerControllerFactory, isDataTransferMessage } from './utils';
import { createLocalStrapiSourceProvider, ILocalStrapiSourceProvider } from '../../providers';
import {
createTransferAssetStreamChunk,
transferAssetStreamChunkByteLength,
} from '../../../utils/transfer-asset-chunk';
import {
createLocalStrapiSourceProvider,
estimateAssetTotals,
ILocalStrapiSourceProvider,
} from '../../providers';
import { ProviderTransferError } from '../../../errors/providers';
import type { IAsset, TransferStage, Protocol } from '../../../../types';
import type { IAsset, StageTotalsEstimate, TransferStage, Protocol } from '../../../../types';
import { Client } from '../../../../types/remote/protocol';
const TRANSFER_KIND = 'pull';
@@ -18,6 +26,7 @@ export interface PullHandler extends Handler {
provider?: ILocalStrapiSourceProvider;
streams?: { [stage in TransferStage]?: Readable };
checksumsEnabled?: boolean;
assertValidTransferAction(action: string): asserts action is PullTransferAction;
@@ -43,6 +52,7 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
proto.cleanup.call(this);
this.streams = {};
this.checksumsEnabled = false;
delete this.provider;
},
@@ -205,11 +215,11 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
batch = [];
};
if (!stream) {
throw new ProviderTransferError(`No available stream found for ${stage}`);
}
try {
if (!stream) {
throw new ProviderTransferError(`No available stream found for ${stage}`);
}
for await (const chunk of stream) {
if (stage !== 'assets') {
batch.push(chunk);
@@ -247,10 +257,21 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
const flushUUID = randomUUID();
await this.createReadableStreamForStep(step);
this.flush(step, flushUUID);
let totals: StageTotalsEstimate | undefined;
if (step === 'assets') {
totals = await estimateAssetTotals(strapi as Core.Strapi);
}
return { ok: true, id: flushUUID };
await this.createReadableStreamForStep(step);
Promise.resolve(this.flush(step, flushUUID)).catch((err: unknown) => {
this.onError(err instanceof Error ? err : new Error(String(err)));
});
return {
ok: true,
id: flushUUID,
...(totals !== undefined ? { totals } : {}),
};
}
if (action === 'end') {
@@ -276,12 +297,10 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
assets: () => {
const assets = this.provider?.createAssetsReadStream();
let batch: Protocol.Client.TransferAssetFlow[] = [];
const checksumsEnabled = this.checksumsEnabled === true;
const batchLength = () => {
return batch.reduce(
(acc, chunk) => (chunk.action === 'stream' ? acc + chunk.data.byteLength : acc),
0
);
return batch.reduce((acc, chunk) => acc + transferAssetStreamChunkByteLength(chunk), 0);
};
const BATCH_MAX_SIZE = 1024 * 1024; // 1MB
@@ -298,19 +317,21 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
async function* generator(stream: Readable) {
let hasStarted = false;
let assetID = '';
let assetChecksum: ReturnType<typeof createHash> | undefined;
for await (const chunk of stream) {
const { stream: assetStream, ...assetData } = chunk as IAsset;
if (!hasStarted) {
assetID = randomUUID();
assetChecksum = checksumsEnabled ? createHash('sha256') : undefined;
// Start the transfer of a new asset
batch.push({ action: 'start', assetID, data: assetData });
hasStarted = true;
}
for await (const assetChunk of assetStream) {
// Add the asset data to the batch
batch.push({ action: 'stream', assetID, data: assetChunk });
assetChecksum?.update(assetChunk);
batch.push(createTransferAssetStreamChunk(assetID, assetChunk));
// if the batch size is bigger than BATCH_MAX_SIZE stream the batch
if (batchLength() >= BATCH_MAX_SIZE) {
@@ -321,7 +342,13 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
// All the asset data has been streamed and gets ready for the next one
hasStarted = false;
batch.push({ action: 'end', assetID });
batch.push({
action: 'end',
assetID,
...(assetChecksum
? { checksum: { algorithm: 'sha256' as const, value: assetChecksum.digest('hex') } }
: {}),
});
yield batch;
batch = [];
}
@@ -343,7 +370,7 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
},
// Commands
async init(this: PullHandler) {
async init(this: PullHandler, params?: Protocol.Client.GetCommandParams<'init'>) {
if (this.transferID || this.provider) {
throw new Error('Transfer already in progress');
}
@@ -351,6 +378,7 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
this.transferID = randomUUID();
this.startedAt = Date.now();
this.checksumsEnabled = params?.checksums === true;
this.streams = {};
@@ -359,7 +387,7 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
getStrapi: () => strapi as Core.Strapi,
});
return { transferID: this.transferID };
return { transferID: this.transferID, checksums: true };
},
async end(
@@ -1,4 +1,4 @@
import { randomUUID } from 'crypto';
import { createHash, randomUUID, type Hash } from 'crypto';
import { Writable, PassThrough } from 'stream';
import type { Core } from '@strapi/types';
@@ -6,6 +6,8 @@ import type { TransferFlow, Step } from '../flows';
import type { TransferStage, IAsset, Protocol } from '../../../../types';
import { ProviderTransferError } from '../../../errors/providers';
import { write } from '../../../utils/writable-async-write';
import { decodeTransferAssetStreamItem } from '../../../utils/transfer-asset-chunk';
import { createLocalStrapiDestinationProvider } from '../../providers';
import { createFlow, DEFAULT_TRANSFER_FLOW } from '../flows';
import { Handler } from './abstract';
@@ -42,13 +44,21 @@ export interface PushHandler extends Handler {
/**
* Holds all the transferred assets for the current transfer handler (one registry per connection)
*/
assets: { [filepath: string]: IAsset & { stream: PassThrough } };
assets: { [assetID: string]: IAsset & { stream: PassThrough } };
/** Incremental checksum state keyed by transfer asset ID (only populated when checksums are enabled). */
assetChecksums?: { [assetID: string]: Hash };
checksumsEnabled?: boolean;
/**
* Ochestrate and manage the transfer messages' ordering
* Orchestrate and manage the transfer messages' ordering
*/
flow?: TransferFlow;
/**
* Interval for periodic destination memory logging during assets stage
*/
memoryLogInterval?: ReturnType<typeof setInterval>;
/**
* Checks that the given action is a valid push transfer action
*/
@@ -105,18 +115,6 @@ export interface PushHandler extends Handler {
assertValidStreamTransferStep(stage: TransferStage): void;
}
const writeAsync = <T>(stream: Writable, data: T) => {
return new Promise<void>((resolve, reject) => {
stream.write(data, (error) => {
if (error) {
reject(error);
}
resolve();
});
});
};
export const createPushController = handlerControllerFactory<Partial<PushHandler>>((proto) => ({
isTransferStarted(this: PushHandler) {
return proto.isTransferStarted.call(this) && this.provider !== undefined;
@@ -146,16 +144,26 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
});
},
cleanup(this: PushHandler) {
if (this.memoryLogInterval) {
clearInterval(this.memoryLogInterval);
delete this.memoryLogInterval;
}
proto.cleanup.call(this);
this.streams = {};
this.assets = {};
this.assetChecksums = {};
this.checksumsEnabled = false;
delete this.flow;
delete this.provider;
},
teardown(this: PushHandler) {
if (this.memoryLogInterval) {
clearInterval(this.memoryLogInterval);
delete this.memoryLogInterval;
}
if (this.provider) {
this.provider.rollback();
}
@@ -338,6 +346,21 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
this.stats[stage] = { started: 0, finished: 0 };
if (stage === 'assets') {
strapi.log.debug(
'[Transfer destination] Assets stage started; sampling memory usage every 5s until stage end'
);
this.memoryLogInterval = setInterval(() => {
const mem = process.memoryUsage();
const stats = this.stats?.assets;
const rssMb = (mem.rss / 1024 / 1024).toFixed(1);
const heapMb = (mem.heapUsed / 1024 / 1024).toFixed(1);
strapi.log.debug(
`[Transfer destination] memory RSS=${rssMb}MB heapUsed=${heapMb}MB | assets started=${stats?.started ?? 0} finished=${stats?.finished ?? 0}`
);
}, 5000);
}
return { ok: true };
}
@@ -356,17 +379,20 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
return this.streamAsset(msg.data);
}
// For all other steps
await Promise.all(
msg.data.map(async (item) => {
this.stats[stage].started += 1;
await writeAsync(stream, item);
this.stats[stage].finished += 1;
})
);
// One objectMode Writable: do not overlap writes.
for (const item of msg.data) {
this.stats[stage].started += 1;
await write(stream, item);
this.stats[stage].finished += 1;
}
}
if (msg.action === 'end') {
if (stage === 'assets' && this.memoryLogInterval) {
clearInterval(this.memoryLogInterval);
delete this.memoryLogInterval;
strapi.log.debug('[Transfer destination] Assets stage ended, stopped memory log');
}
this.unlockTransferStep(stage);
const stream = this.streams?.[stage];
@@ -424,29 +450,66 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
if (action === 'start') {
this.stats.assets.started += 1;
this.assets[assetID] = { ...item.data, stream: new PassThrough() };
writeAsync(assetsStream, this.assets[assetID]);
}
if (this.checksumsEnabled) {
this.assetChecksums ??= {};
this.assetChecksums[assetID] = createHash('sha256');
}
const filename = item.data?.filename ?? assetID;
strapi.log.debug(
`[Transfer destination] Asset start #${this.stats.assets.started} id=${assetID} filename=${filename}`
);
// Wait for the assets stage to accept this row (same pattern as remote-source).
await write(assetsStream, this.assets[assetID]);
} else if (action === 'stream' || action === 'end') {
if (!this.assets[assetID]) {
throw new ProviderTransferError(
`No asset "${assetID}" for ${action} action; send start before stream/end`
);
}
if (action === 'stream') {
// The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
// We need to transform it back into a Buffer instance
const rawBuffer = item.data as unknown as { type: 'Buffer'; data: Uint8Array };
const chunk = Buffer.from(rawBuffer.data);
await writeAsync(this.assets[assetID].stream, chunk);
}
if (action === 'end') {
await new Promise<void>((resolve, reject) => {
const { stream: assetStream } = this.assets[assetID];
assetStream
.on('close', () => {
this.stats.assets.finished += 1;
delete this.assets[assetID];
resolve();
})
.on('error', reject)
.end();
});
if (action === 'stream') {
const chunk = decodeTransferAssetStreamItem(item);
this.assetChecksums?.[assetID]?.update(chunk);
await write(this.assets[assetID].stream, chunk);
} else {
if (this.checksumsEnabled) {
if (!item.checksum) {
throw new ProviderTransferError(`Missing checksum for asset "${assetID}"`);
}
if (item.checksum.algorithm !== 'sha256') {
throw new ProviderTransferError(
`Unsupported checksum algorithm "${item.checksum.algorithm}" for asset ${assetID}`
);
}
const checksum = this.assetChecksums?.[assetID]?.digest('hex');
if (!checksum || checksum !== item.checksum.value) {
throw new ProviderTransferError(
`Checksum mismatch for asset "${assetID}" (expected ${item.checksum.value}, got ${checksum ?? 'none'})`
);
}
}
if (this.assetChecksums?.[assetID]) {
delete this.assetChecksums[assetID];
}
strapi.log.debug(
`[Transfer destination] Asset end id=${assetID} (finished=${this.stats.assets.finished + 1}/${this.stats.assets.started})`
);
await new Promise<void>((resolve, reject) => {
const { stream: assetStream } = this.assets[assetID];
assetStream
.on('close', () => {
this.stats.assets.finished += 1;
delete this.assets[assetID];
resolve();
})
.on('error', reject)
.end();
});
}
} else {
throw new ProviderTransferError(
`Invalid asset flow action: ${String((item as { action?: unknown }).action)}`
);
}
}
},
@@ -476,6 +539,8 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
this.startedAt = Date.now();
this.assets = {};
this.assetChecksums = {};
this.checksumsEnabled = params?.checksums === true;
this.streams = {};
this.stats = {
assets: { started: 0, finished: 0 },
@@ -487,7 +552,8 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
this.flow = createFlow(DEFAULT_TRANSFER_FLOW);
this.provider = createLocalStrapiDestinationProvider({
...params.options,
strategy: params?.options?.strategy ?? 'restore',
restore: params?.options?.restore ?? {},
autoDestroy: false,
getStrapi: () => strapi as Core.Strapi,
});
@@ -497,7 +563,7 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
strapi.log.warn(message);
};
return { transferID: this.transferID };
return { transferID: this.transferID, checksums: true };
},
async status(this: PushHandler) {
@@ -10,6 +10,7 @@ import { ProviderError, ProviderTransferError } from '../../../errors/providers'
import { VALID_TRANSFER_COMMANDS, ValidTransferCommand } from './constants';
import { TransferMethod } from '../constants';
import { createDiagnosticReporter } from '../../../utils/diagnostic';
import { stringifyTransferWebSocketPayload } from '../../../utils/transfer-websocket-json';
type WSCallback = (client: WebSocket, request: IncomingMessage) => void;
@@ -245,7 +246,7 @@ export const handlerControllerFactory =
details = e.details;
}
const payload = JSON.stringify({
const envelope: Record<string, unknown> = {
uuid,
data: data ?? null,
error: e
@@ -255,7 +256,9 @@ export const handlerControllerFactory =
details,
}
: null,
});
};
const payload = stringifyTransferWebSocketPayload(envelope);
this.send(payload, (error) => (error ? reject(error) : resolve()));
});
@@ -268,7 +271,7 @@ export const handlerControllerFactory =
return new Promise((resolve, reject) => {
const uuid = randomUUID();
const payload = JSON.stringify({ uuid, data: message });
const payload = stringifyTransferWebSocketPayload({ uuid, data: message });
this.send(payload, (error) => {
if (error) {
@@ -0,0 +1,169 @@
import { Readable, Writable } from 'stream';
import { pipeline } from 'stream/promises';
import { filter, map, collect } from '../stream';
describe('stream utils', () => {
describe('filter', () => {
test('propagates backpressure: source pauses when downstream is slow', async () => {
const itemCount = 20;
const data = Array.from({ length: itemCount }, (_, i) => ({ id: i }));
let sourcePaused = false;
const source = Readable.from(data, { objectMode: true });
const originalPause = source.pause.bind(source);
source.pause = function () {
sourcePaused = true;
return originalPause();
};
const slowConsumer = new Writable({
objectMode: true,
highWaterMark: 1,
write(chunk, _encoding, callback) {
setTimeout(callback, 5);
},
});
const filtered = filter<{ id: number }>((x) => x.id % 2 === 0);
await pipeline(source, filtered, slowConsumer);
expect(sourcePaused).toBe(true);
});
test('does not retain references to discarded chunks', async () => {
const data = Array.from({ length: 100 }, (_, i) => ({ id: i }));
const collected: Array<{ id: number }> = [];
const source = Readable.from(data, { objectMode: true });
const filtered = filter<{ id: number }>((x) => x.id % 2 === 0);
const writer = new Writable({
objectMode: true,
write(chunk, _encoding, callback) {
collected.push(chunk);
callback();
},
});
await pipeline(source, filtered, writer);
expect(collected).toHaveLength(50);
expect(collected.map((x) => x.id)).toEqual(
data.filter((_, i) => i % 2 === 0).map((x) => x.id)
);
});
test('stream is destroyed after pipeline completes', async () => {
const source = Readable.from([1, 2, 3], { objectMode: true });
const filtered = filter<number>(() => true);
const dest = new Writable({
objectMode: true,
write(chunk, _enc, cb) {
cb();
},
});
await pipeline(source, filtered, dest);
expect(source.destroyed).toBe(true);
expect(filtered.destroyed).toBe(true);
expect(dest.destroyed).toBe(true);
});
});
describe('map', () => {
test('propagates backpressure: source pauses when downstream is slow', async () => {
const itemCount = 20;
const data = Array.from({ length: itemCount }, (_, i) => i);
let sourcePaused = false;
const source = Readable.from(data, { objectMode: true });
const originalPause = source.pause.bind(source);
source.pause = function () {
sourcePaused = true;
return originalPause();
};
const slowConsumer = new Writable({
objectMode: true,
highWaterMark: 1,
write(_chunk, _encoding, callback) {
setTimeout(callback, 5);
},
});
const mapped = map<number, number>((x) => x * 2);
await pipeline(source, mapped, slowConsumer);
expect(sourcePaused).toBe(true);
});
test('does not retain references to all chunks', async () => {
const data = Array.from({ length: 100 }, (_, i) => i);
const collected: number[] = [];
const source = Readable.from(data, { objectMode: true });
const mapped = map<number, number>((x) => x + 1);
const writer = new Writable({
objectMode: true,
write(chunk, _encoding, callback) {
collected.push(chunk);
callback();
},
});
await pipeline(source, mapped, writer);
expect(collected).toHaveLength(100);
expect(collected).toEqual(data.map((x) => x + 1));
});
test('stream is destroyed after pipeline completes', async () => {
const source = Readable.from([1, 2, 3], { objectMode: true });
const mapped = map<number, number>((x) => x);
const dest = new Writable({
objectMode: true,
write(chunk, _enc, cb) {
cb();
},
});
await pipeline(source, mapped, dest);
expect(source.destroyed).toBe(true);
expect(mapped.destroyed).toBe(true);
expect(dest.destroyed).toBe(true);
});
});
describe('collect', () => {
test('destroys stream when options.destroy is true (default)', async () => {
const source = Readable.from([1, 2, 3], { objectMode: true });
const result = await collect<number>(source);
expect(result).toEqual([1, 2, 3]);
expect(source.destroyed).toBe(true);
});
test('with destroy false still resolves with correct data', async () => {
const source = Readable.from([1, 2, 3], { objectMode: true });
const result = await collect<number>(source, { destroy: false });
expect(result).toEqual([1, 2, 3]);
});
test('resolves on close with chunks received so far', async () => {
const source = new Readable({
objectMode: true,
read() {
this.push(1);
this.push(2);
this.destroy();
},
});
const result = await collect<number>(source);
expect(result).toEqual([1, 2]);
});
test('rejects on stream error', async () => {
const source = new Readable({
objectMode: true,
read() {
this.destroy(new Error('test error'));
},
});
await expect(collect(source)).rejects.toThrow('test error');
});
});
});
@@ -0,0 +1,139 @@
import {
createTransferAssetStreamChunk,
decodeTransferAssetStreamData,
decodeTransferAssetStreamItem,
transferAssetStreamChunkByteLength,
} from '../transfer-asset-chunk';
import { replacerForTransferWebSocket } from '../transfer-websocket-json';
describe('transfer-asset-chunk', () => {
const bytes = Buffer.from([0xde, 0xad, 0xbe, 0xef]);
test('createTransferAssetStreamChunk + decodeTransferAssetStreamItem round-trip', () => {
const item = createTransferAssetStreamChunk('asset-1', bytes);
expect(item).toEqual(
expect.objectContaining({ action: 'stream', assetID: 'asset-1', encoding: 'base64' })
);
expect(decodeTransferAssetStreamItem(item)).toEqual(bytes);
});
test('decodeTransferAssetStreamItem: legacy Buffer JSON item', () => {
const item = {
action: 'stream' as const,
assetID: 'a',
data: { type: 'Buffer' as const, data: Array.from(bytes) },
};
expect(decodeTransferAssetStreamItem(item)).toEqual(bytes);
});
test('decodeTransferAssetStreamData: explicit encoding base64', () => {
expect(decodeTransferAssetStreamData(bytes.toString('base64'), 'base64')).toEqual(bytes);
});
test('decodeTransferAssetStreamData: legacy Buffer JSON shape (client push / old pull)', () => {
const legacy = { type: 'Buffer' as const, data: Array.from(bytes) };
expect(decodeTransferAssetStreamData(legacy)).toEqual(bytes);
});
test('decodeTransferAssetStreamData: Buffer instance (in-process)', () => {
expect(decodeTransferAssetStreamData(Buffer.from(bytes))).toEqual(bytes);
});
test('decodeTransferAssetStreamData: base64 string without encoding flag', () => {
const b64 = bytes.toString('base64');
expect(decodeTransferAssetStreamData(b64)).toEqual(bytes);
});
test('wire compatibility: Buffer property stringifies as legacy JSON (Node toJSON, not replacer)', () => {
const item = { action: 'stream' as const, assetID: 'a', data: Buffer.from(bytes) };
const wire = JSON.parse(JSON.stringify(item, replacerForTransferWebSocket)) as {
data: unknown;
};
expect(wire.data).toEqual(expect.objectContaining({ type: 'Buffer' }));
expect(decodeTransferAssetStreamData(wire.data)).toEqual(bytes);
});
test('wire compatibility: Uint8Array → WS replacer → base64 string → decode', () => {
const item = { action: 'stream' as const, assetID: 'a', data: new Uint8Array(bytes) };
const wire = JSON.parse(JSON.stringify(item, replacerForTransferWebSocket)) as {
data: unknown;
};
expect(typeof wire.data).toBe('string');
expect(decodeTransferAssetStreamData(wire.data)).toEqual(bytes);
});
test('wire compatibility: stream item legacy JSON shape survives parse unchanged', () => {
const item = {
action: 'stream' as const,
assetID: 'a',
data: { type: 'Buffer' as const, data: Array.from(bytes) },
};
const wire = JSON.parse(JSON.stringify(item)) as { data: unknown };
expect(decodeTransferAssetStreamData(wire.data)).toEqual(bytes);
});
test('wire compatibility: explicit pull shape encoding+string', () => {
const item = {
action: 'stream' as const,
assetID: 'a',
encoding: 'base64' as const,
data: bytes.toString('base64'),
};
const wire = JSON.parse(JSON.stringify(item)) as typeof item;
expect(decodeTransferAssetStreamData(wire.data, wire.encoding)).toEqual(bytes);
});
test('transferAssetStreamChunkByteLength: base64 string (with or without encoding flag)', () => {
const b64 = bytes.toString('base64');
expect(
transferAssetStreamChunkByteLength({
action: 'stream',
encoding: 'base64',
data: b64,
})
).toBe(Math.floor((b64.length * 3) / 4));
expect(
transferAssetStreamChunkByteLength({
action: 'stream',
data: b64,
})
).toBe(Math.floor((b64.length * 3) / 4));
});
test('transferAssetStreamChunkByteLength: Buffer and non-stream', () => {
expect(
transferAssetStreamChunkByteLength({
action: 'stream',
data: bytes,
})
).toBe(4);
expect(transferAssetStreamChunkByteLength({ action: 'end' })).toBe(0);
});
test('decodeTransferAssetStreamData: encoding base64 with legacy object falls back (no throw)', () => {
const legacy = { type: 'Buffer' as const, data: Array.from(bytes) };
expect(decodeTransferAssetStreamData(legacy, 'base64')).toEqual(bytes);
});
test('decodeTransferAssetStreamItem: encoding base64 with legacy payload', () => {
const item = {
action: 'stream' as const,
assetID: 'a',
encoding: 'base64' as const,
data: { type: 'Buffer' as const, data: Array.from(bytes) },
};
expect(decodeTransferAssetStreamItem(item)).toEqual(bytes);
});
test('transferAssetStreamChunkByteLength: legacy Buffer JSON shape', () => {
const legacy = { type: 'Buffer' as const, data: Array.from(bytes) };
expect(
transferAssetStreamChunkByteLength({
action: 'stream',
data: legacy,
})
).toBe(bytes.length);
});
});
@@ -0,0 +1,133 @@
import {
replacerForTransferWebSocket,
stringifyTransferWebSocketPayload,
} from '../transfer-websocket-json';
describe('stringifyTransferWebSocketPayload', () => {
test('raw JSON.stringify throws on BigInt; wire helper encodes nested bigint metadata as decimal strings', () => {
expect(() => JSON.stringify({ size: 9007199254740993n })).toThrow();
const s = stringifyTransferWebSocketPayload({
uuid: 'u',
type: 'transfer',
kind: 'step',
action: 'stream',
step: 'assets',
data: [
{
action: 'start',
assetID: '1',
metadata: { size: 9007199254740993n, name: 'blob.bin' },
},
],
} as Record<string, unknown>);
expect(s).toContain('"size":"9007199254740993"');
expect(s).toContain('"name":"blob.bin"');
});
test('circular nested data throws TypeError with context (peer cannot decode cycles)', () => {
const cyclic: Record<string, unknown> = { tag: 'x' };
cyclic.self = cyclic;
expect(() =>
stringifyTransferWebSocketPayload({
uuid: 'u',
type: 'transfer',
data: cyclic,
} as Record<string, unknown>)
).toThrow(TypeError);
expect(() =>
stringifyTransferWebSocketPayload({
uuid: 'u',
type: 'transfer',
data: cyclic,
} as Record<string, unknown>)
).toThrow(/could not be serialized to JSON/);
});
test('upload-like nested metadata (unicode, null, empty object, deep keys) serializes', () => {
const s = stringifyTransferWebSocketPayload({
uuid: 'u',
type: 'transfer',
kind: 'step',
action: 'stream',
step: 'assets',
data: {
action: 'start',
assetID: 'file-48',
metadata: {
name: 'café 文件 🗂️.bin',
caption: null,
alternativeText: '',
provider_metadata: { foo: { bar: [1, 2, 3] } },
},
},
} as Record<string, unknown>);
expect(typeof s).toBe('string');
const roundTrip = JSON.parse(s) as { data: { metadata: { name: string } } };
expect(roundTrip.data.metadata.name).toContain('café');
});
test('enumerable toJSON on the root object makes raw JSON.stringify return undefined (ws.send would throw)', () => {
const evil = {
type: 'transfer',
kind: 'step',
action: 'stream',
step: 'assets',
data: [],
toJSON() {
return undefined;
},
};
const payload = { ...evil, uuid: 'test-uuid' };
expect(JSON.stringify(payload, replacerForTransferWebSocket)).toBeUndefined();
});
test('stringifyTransferWebSocketPayload strips enumerable toJSON so serialization succeeds (fixes ws.send(undefined))', () => {
const evil = {
type: 'transfer',
kind: 'step',
action: 'stream',
step: 'assets',
data: [],
toJSON() {
return undefined;
},
};
const payload = { ...evil, uuid: 'test-uuid' } as Record<string, unknown>;
const s = stringifyTransferWebSocketPayload(payload);
expect(typeof s).toBe('string');
expect(s).toContain('"uuid":"test-uuid"');
expect(s).toContain('"type":"transfer"');
});
test('replacer encodes Uint8Array values as base64 strings (nested Buffer may use Buffer.toJSON before replacer)', () => {
const payload = {
uuid: 'u',
raw: new Uint8Array([0xde, 0xad]),
} as Record<string, unknown>;
const s = stringifyTransferWebSocketPayload(payload);
expect(s).toContain('3q0=');
});
test('nested plain objects do not need root toJSON strip; only root spread could copy enumerable toJSON', () => {
const inner = {
id: 48,
formats: { large: { url: '/u/x' } },
toJSON() {
return { id: 48, note: 'orm-shaped' };
},
};
const s = stringifyTransferWebSocketPayload({
uuid: 'u',
type: 'transfer',
data: { file: inner },
} as Record<string, unknown>);
expect(typeof s).toBe('string');
expect(s).toContain('"note":"orm-shaped"');
});
});
@@ -0,0 +1,83 @@
import { Writable } from 'stream';
import { write } from '../writable-async-write';
describe('writable-async-write (write)', () => {
const created: Writable[] = [];
afterEach(() => {
while (created.length) {
created.pop()?.destroy();
}
});
test('waits for drain when write returns false (slow consumer, low highWaterMark)', async () => {
let drainCount = 0;
const dest = new Writable({
objectMode: true,
highWaterMark: 1,
write(_chunk, _encoding, callback) {
setTimeout(callback, 2);
},
});
created.push(dest);
dest.on('drain', () => {
drainCount += 1;
});
for (let i = 0; i < 12; i += 1) {
await write(dest, { n: i });
}
expect(drainCount).toBeGreaterThan(0);
});
test('resolves when write returns true (room in buffer)', async () => {
const dest = new Writable({
objectMode: true,
highWaterMark: 100,
write(_chunk, _encoding, callback) {
callback();
},
});
created.push(dest);
await expect(write(dest, { x: 1 })).resolves.toBeUndefined();
});
test('rejects when write callback receives an error', async () => {
const dest = new Writable({
objectMode: true,
write(_chunk, _encoding, callback) {
callback(new Error('sink error'));
},
});
created.push(dest);
await expect(write(dest, { x: 1 })).rejects.toThrow('sink error');
});
test('does not hang when writable is destroyed while awaiting drain (race with stream finished)', async () => {
const dest = new Writable({
objectMode: true,
highWaterMark: 1,
write(_chunk, _encoding, callback) {
setTimeout(callback, 2);
},
});
created.push(dest);
const run = async () => {
for (let i = 0; i < 30; i += 1) {
if (i === 8) {
setImmediate(() => {
dest.destroy(new Error('aborted'));
});
}
await write(dest, { n: i });
}
};
await expect(run()).rejects.toThrow('aborted');
});
});
@@ -1,4 +1,6 @@
export * as encryption from './encryption';
export * from './transfer-asset-chunk';
export { write } from './writable-async-write';
export * as stream from './stream';
export * as json from './json';
export * as schema from './schema';
+50 -10
View File
@@ -57,16 +57,56 @@ export const collect = <T = unknown>(
const chunks: T[] = [];
return new Promise((resolve, reject) => {
stream
.on('close', () => resolve(chunks))
.on('error', reject)
.on('data', (chunk) => chunks.push(chunk))
.on('end', () => {
if (options.destroy) {
stream.destroy();
}
let settled = false;
resolve(chunks);
});
const cleanup = () => {
stream.removeListener('data', onData);
stream.removeListener('end', onEnd);
stream.removeListener('close', onClose);
stream.removeListener('error', onError);
};
const finishResolve = () => {
if (settled) {
return;
}
settled = true;
cleanup();
resolve(chunks);
};
const finishReject = (err: unknown) => {
if (settled) {
return;
}
settled = true;
cleanup();
reject(err);
};
const onData = (chunk: T) => {
chunks.push(chunk);
};
const onEnd = () => {
if (options.destroy) {
stream.destroy();
}
finishResolve();
};
const onClose = () => {
// Handles streams that emit `close` without `end` (e.g. `destroy()` in `_read`).
finishResolve();
};
const onError = (err: Error) => {
finishReject(err);
};
stream.on('data', onData);
stream.on('end', onEnd);
stream.on('close', onClose);
stream.on('error', onError);
});
};
@@ -0,0 +1,114 @@
/**
* Canonical **outbound** asset chunk for WebSocket JSON (push and pull).
* Base64 string `data` keeps `JSON.parse` heap bounded vs `{ type: 'Buffer', data: [n,…] }`.
*/
export function createTransferAssetStreamChunk(
assetID: string,
chunk: Buffer | Uint8Array
): { action: 'stream'; assetID: string; encoding: 'base64'; data: string } {
if (chunk == null) {
throw new TypeError(
'Asset stream yielded a null/undefined chunk; refusing to encode (would trigger Buffer.from(undefined))'
);
}
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
return {
action: 'stream',
assetID,
encoding: 'base64',
data: buffer.toString('base64'),
};
}
/**
* Decode a stream item from `TransferAssetFlow` after `JSON.parse` (shared by push + pull handlers
* and the remote source provider).
*/
export function decodeTransferAssetStreamItem(item: {
action: 'stream';
data: unknown;
encoding?: 'base64';
}): Buffer {
return decodeTransferAssetStreamData(
item.data,
item.encoding === 'base64' ? 'base64' : undefined
);
}
const getLegacyBufferJsonData = (value: unknown): Uint8Array | readonly number[] | null => {
if (!value || typeof value !== 'object' || !('type' in value)) {
return null;
}
if ((value as { type: unknown }).type !== 'Buffer') {
return null;
}
const raw = (value as { data?: unknown }).data;
if (Array.isArray(raw) || ArrayBuffer.isView(raw)) {
return raw as Uint8Array | readonly number[];
}
return null;
};
/**
* Decode binary payload for `TransferAssetFlow` `action: 'stream'` after JSON.parse.
*
* Supported shapes (receivers should accept all of these):
* - **String `data`:** preferred wire form (`createTransferAssetStreamChunk` / `encoding: 'base64'`).
* - **`{ type: 'Buffer', data: number[] | TypedArray }`:** legacy `Buffer.toJSON()` from default
* `JSON.stringify` (older clients/servers).
* - **`Buffer` instance:** in-process only.
*
* Note: Nodes `JSON.stringify` runs `Buffer.toJSON()` before any replacer, so nested `Buffer`
* values become the legacy object unless you pass a string (use `createTransferAssetStreamChunk`).
*/
export function decodeTransferAssetStreamData(data: unknown, encoding?: 'base64'): Buffer {
if (encoding === 'base64' && typeof data === 'string') {
return Buffer.from(data, 'base64');
}
// `encoding: 'base64'` with a non-string payload (or no encoding) uses the same fallbacks as
// legacy peers — avoids throwing when flags and payload disagree.
if (Buffer.isBuffer(data)) {
return Buffer.from(data);
}
const legacyBufferData = getLegacyBufferJsonData(data);
if (legacyBufferData) {
return Buffer.from(legacyBufferData);
}
// Wire base64 string (pull generator and any other path that stringifies a string payload).
if (typeof data === 'string') {
return Buffer.from(data, 'base64');
}
throw new TypeError('Invalid transfer asset stream chunk payload');
}
/** Approximate decoded byte size for batching (pull asset generator). */
export function transferAssetStreamChunkByteLength(chunk: {
action: string;
data?: unknown;
encoding?: 'base64';
}): number {
if (chunk.action !== 'stream') {
return 0;
}
if (typeof chunk.data === 'string') {
return Math.floor((chunk.data.length * 3) / 4);
}
if (Buffer.isBuffer(chunk.data)) {
return chunk.data.byteLength;
}
const legacyBufferData = getLegacyBufferJsonData(chunk.data);
if (legacyBufferData) {
if (Array.isArray(legacyBufferData)) {
return legacyBufferData.length;
}
if (ArrayBuffer.isView(legacyBufferData)) {
return legacyBufferData.byteLength;
}
}
return 0;
}
@@ -0,0 +1,72 @@
/**
* Shared `JSON.stringify` replacer for data-transfer WebSocket frames (push and pull).
*
* Default `JSON.stringify` uses `Buffer.toJSON()` → `{ type: 'Buffer', data: [n,n,...] }`, which
* allocates a large array on the peer during `JSON.parse`. Encode binary values as compact base64 strings instead.
*
* Note: Node runs `Buffer.prototype.toJSON` before the replacer sees a `Buffer` property, so the
* replacer receives `{ type: 'Buffer', data: [...] }` unless the value is already a string (see
* `createTransferAssetStreamChunk` in `transfer-asset-chunk.ts`).
*/
export const replacerForTransferWebSocket = (_key: string, value: unknown): unknown => {
/** `JSON.stringify` throws on bigint; upload metadata or ORM fields may surface as BigInt. */
if (typeof value === 'bigint') {
return value.toString();
}
if (Buffer.isBuffer(value)) {
return value.toString('base64');
}
if (value instanceof Uint8Array) {
const { buffer, byteOffset, byteLength } = value;
if (buffer == null) {
throw new TypeError(
'Invalid Uint8Array in transfer payload (missing underlying ArrayBuffer); cannot encode for WebSocket'
);
}
return Buffer.from(buffer, byteOffset, byteLength).toString('base64');
}
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
const v = value as NodeJS.TypedArray;
const { buffer, byteOffset, byteLength } = v;
if (buffer == null) {
throw new TypeError(
'Invalid typed array in transfer payload (missing underlying ArrayBuffer); cannot encode for WebSocket'
);
}
return Buffer.from(buffer, byteOffset, byteLength).toString('base64');
}
return value;
};
/**
* `JSON.stringify` invokes an own enumerable `toJSON` on the root value before replacers run. If that
* method returns `undefined`, the whole `JSON.stringify` result is `undefined`, and `ws.send(undefined)`
* throws ("The first argument must be of type string or an instance of Buffer... Received undefined").
* Spreading transfer messages (`{ ...message, uuid }`) can copy an enumerable `toJSON` from user / ORM
* objects onto the wire payload — strip it on the root object we control.
*/
export function stripRootToJSONMethod(payload: Record<string, unknown>): void {
if (typeof payload.toJSON === 'function') {
delete payload.toJSON;
}
}
/**
* Serialize a transfer WebSocket envelope. Never returns `undefined` (unlike raw `JSON.stringify`).
*/
export function stringifyTransferWebSocketPayload(payload: Record<string, unknown>): string {
stripRootToJSONMethod(payload);
let s: string | undefined;
try {
s = JSON.stringify(payload, replacerForTransferWebSocket);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
throw new TypeError(`Transfer WebSocket payload could not be serialized to JSON: ${message}`);
}
if (typeof s !== 'string') {
throw new TypeError(
'Transfer WebSocket payload could not be serialized to JSON (result was undefined). Check for Symbol or other non-JSON values on the root payload.'
);
}
return s;
}
@@ -0,0 +1,57 @@
import { once } from 'node:events';
import { finished } from 'node:stream/promises';
import type { Writable } from 'stream';
/**
* Async helper for application code that `await`s sequential writes to a `Writable`.
*
* 1. Waits until `writable.write` invokes its callback (chunk accepted / `_write` finished).
* 2. If `write()` returned `false` **and** `writable.writableNeedDrain` is still true after the
* callback, waits for `'drain'`.
*
* We check both: the return value tells us backpressure was signaled; `writableNeedDrain` avoids
* awaiting `'drain'` when it already fired before we subscribed (would otherwise hang forever).
*
* While waiting for `'drain'`, we also race {@link finished} so destroying the writable (e.g. abort)
* cannot leave this promise pending forever.
*/
export async function write(writable: Writable, chunk: unknown): Promise<void> {
let flushed = true;
await new Promise<void>((resolve, reject) => {
let settled = false;
const finish = (fn: () => void) => {
if (settled) {
return;
}
settled = true;
writable.off('error', onError);
fn();
};
const onError = (err: Error) => {
finish(() => reject(err));
};
writable.once('error', onError);
flushed = writable.write(chunk, (err) => {
if (err) {
// Do not reject or remove `error` here: Node may emit `error` after this callback, and
// clearing the listener first would leave that emission unhandled.
setImmediate(() => {
if (!settled) {
finish(() => reject(err));
}
});
return;
}
finish(() => resolve());
});
});
if (!flushed && writable.writableNeedDrain) {
// Without `finished`, awaiting only `drain` can hang forever if the writable is destroyed first.
await Promise.race([
once(writable, 'drain'),
finished(writable, { readable: false, writable: true }),
]);
}
}
+8
View File
@@ -4,6 +4,8 @@ import type {
IProviderTransferResults,
ISourceProviderTransferResults,
MaybePromise,
StageTotalsEstimate,
TransferStage,
} from './utils';
import type { IMetadata } from './common-entities';
import type { IDiagnosticReporter } from '../src/utils/diagnostic';
@@ -31,6 +33,12 @@ export interface IProvider {
export interface ISourceProvider extends IProvider {
results?: ISourceProviderTransferResults;
/**
* Optional totals for a stage. Called by the engine after the stage read stream is created and before `stage::start`.
* Used for CLI progress (e.g. bytes/count remaining, ETA). Omit or return null when unknown (older remotes, file providers).
*/
getStageTotals?(stage: TransferStage): MaybePromise<StageTotalsEstimate | null | undefined>;
createEntitiesReadStream?(): MaybePromise<Readable>;
createLinksReadStream?(): MaybePromise<Readable>;
createAssetsReadStream?(): MaybePromise<Readable>;
@@ -14,8 +14,9 @@ export type GetCommandParams<T extends Command> = {
export type InitCommand = CreateCommand<
'init',
{
options: Pick<ILocalStrapiDestinationProviderOptions, 'strategy' | 'restore'>;
options?: Pick<ILocalStrapiDestinationProviderOptions, 'strategy' | 'restore'>;
transfer: TransferMethod;
checksums?: boolean;
}
>;
export type TransferKind = InitCommand['params']['transfer'];
@@ -6,8 +6,19 @@ export type CreateTransferMessage<T extends string, U = unknown> = {
transferID: string;
} & U;
export type TransferAssetChecksum = {
algorithm: 'sha256';
value: string;
};
export type TransferAssetFlow = { assetID: string } & (
| { action: 'start'; data: Omit<IAsset, 'stream'> }
| { action: 'stream'; data: Buffer }
| { action: 'end' }
/** Legacy in-process / default JSON: Buffer serializes to `{ type: 'Buffer'; data: number[] }` on the wire. */
| { action: 'stream'; data: Buffer; encoding?: undefined }
/**
* Canonical wire form for asset bytes (push + pull). Built with `createTransferAssetStreamChunk`.
* Decoders also accept legacy Buffer JSON and plain base64 strings without `encoding`.
*/
| { action: 'stream'; data: string; encoding?: 'base64' }
| { action: 'end'; checksum?: TransferAssetChecksum }
);
@@ -11,7 +11,7 @@ export type Message<T = unknown> = {
// Successful
export type OKMessage = Message<{ ok: true }>;
export type InitMessage = Message<{ transferID: string }>;
export type InitMessage = Message<{ transferID: string; checksums?: boolean }>;
export type EndMessage = OKMessage;
export type StatusMessage = Message<
| { active: true; kind: TransferKind; startedAt: number; elapsed: number }
+26 -11
View File
@@ -63,21 +63,36 @@ export type TransferFilters = {
/*
* Progress
*/
export type TransferProgress = {
[key in TransferStage]?: {
count: number;
bytes: number;
startTime: number;
endTime?: number;
aggregates?: {
[key: string]: {
count: number;
bytes: number;
};
export type StageTotalsEstimate = {
totalBytes?: number;
totalCount?: number;
};
/**
* Per-stage running progress. For `assets`, optional totals describe the full transfer when known
* (same basis as streamed data: one count per IAsset including formats; bytes = sum of binary file sizes).
*/
export type StageProgress = {
count: number;
bytes: number;
startTime: number;
endTime?: number;
/** Present when the source can estimate full stage size before/during streaming (e.g. asset library). */
totalBytes?: number;
/** Present when the source can estimate how many items the stage will emit. */
totalCount?: number;
aggregates?: {
[key: string]: {
count: number;
bytes: number;
};
};
};
export type TransferProgress = {
[key in TransferStage]?: StageProgress;
};
export interface ITransferResults<S extends ISourceProvider, D extends IDestinationProvider> {
source?: S['results'];
destination?: D['results'];
@@ -1,5 +1,6 @@
import * as mockDataTransfer from '@strapi/data-transfer';
import * as dataTransferUtils from '../../../utils/data-transfer';
import transferAction from '../action';
import { expectExit } from '../../__tests__/commands.test.utils';
@@ -9,14 +10,15 @@ jest.mock('../../../utils/data-transfer', () => {
getTransferTelemetryPayload: jest.fn().mockReturnValue({}),
loadersFactory: jest.fn().mockReturnValue({ updateLoader: jest.fn() }),
formatDiagnostic: jest.fn(),
createStrapiInstance() {
return {
telemetry: {
send: jest.fn(),
},
contentTypes: {},
};
},
createStrapiInstance: jest.fn(async () => ({
config: {
get: jest.fn(),
},
telemetry: {
send: jest.fn(),
},
contentTypes: {},
})),
getDefaultExportName: jest.fn(() => 'default'),
buildTransferTable: jest.fn(() => {
return {
@@ -34,13 +36,13 @@ jest.mock('../../../utils/data-transfer', () => {
// mock data transfer
jest.mock('@strapi/data-transfer', () => {
const acutal = jest.requireActual('@strapi/data-transfer');
const actual = jest.requireActual('@strapi/data-transfer');
return {
...acutal,
...actual,
strapi: {
...acutal.strapi,
...actual.strapi,
providers: {
...acutal.strapi.providers,
...actual.strapi.providers,
createLocalStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testLocalSource' }),
createLocalStrapiDestinationProvider: jest.fn().mockReturnValue({ name: 'testLocalDest' }),
createRemoteStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testRemoteSource' }),
@@ -50,19 +52,26 @@ jest.mock('@strapi/data-transfer', () => {
},
},
engine: {
...acutal.engine,
...actual.engine,
createTransferEngine() {
const handlers: Record<string, Array<(...args: unknown[]) => void>> = {};
const stream = {
on(event: string, fn: (...args: unknown[]) => void) {
(handlers[event] ||= []).push(fn);
return stream;
},
};
return {
transfer: jest.fn(() => {
transfer: jest.fn(async () => {
// action.ts clears setInterval on this event; real stream emits after transfer.
handlers['transfer::finish']?.forEach((fn) => fn());
return {
engine: {},
};
}),
progress: {
on: jest.fn(),
stream: {
on: jest.fn(),
},
stream,
},
sourceProvider: { name: 'testSource' },
destinationProvider: { name: 'testDestination' },
@@ -82,10 +91,16 @@ describe('Transfer', () => {
// mock command utils
// console spies
jest.spyOn(console, 'log').mockImplementation(() => {});
jest.spyOn(console, 'warn').mockImplementation(() => {});
jest.spyOn(console, 'info').mockImplementation(() => {});
jest.spyOn(console, 'error').mockImplementation(() => {});
beforeAll(() => {
jest.spyOn(console, 'log').mockImplementation(() => {});
jest.spyOn(console, 'warn').mockImplementation(() => {});
jest.spyOn(console, 'info').mockImplementation(() => {});
jest.spyOn(console, 'error').mockImplementation(() => {});
});
afterAll(() => {
jest.restoreAllMocks();
});
const destinationUrl = new URL('http://one.localhost/admin');
const destinationToken = 'test-token';
@@ -95,6 +110,15 @@ describe('Transfer', () => {
beforeEach(() => {
jest.clearAllMocks();
(dataTransferUtils.createStrapiInstance as jest.Mock).mockImplementation(async () => ({
config: {
get: jest.fn(),
},
telemetry: {
send: jest.fn(),
},
contentTypes: {},
}));
});
it('exits with error when no --to or --from is provided', async () => {
@@ -172,6 +196,40 @@ describe('Transfer', () => {
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
).toHaveBeenCalled();
});
it('passes verifyChecksums to remote destination when --checksums is enabled', async () => {
await expectExit(0, async () => {
await transferAction({
from: undefined,
to: destinationUrl,
toToken: destinationToken,
checksums: true,
} as any);
});
expect(
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
).toHaveBeenCalledWith(
expect.objectContaining({
verifyChecksums: true,
})
);
});
it('does not pass verifyChecksums to remote destination when checksums are disabled', async () => {
await expectExit(0, async () => {
await transferAction({
from: undefined,
to: destinationUrl,
toToken: destinationToken,
checksums: false,
} as any);
});
expect(
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
).toHaveBeenCalledWith(expect.not.objectContaining({ verifyChecksums: true }));
});
});
describe('--from', () => {
@@ -210,6 +268,74 @@ describe('Transfer', () => {
);
});
it('passes verifyChecksums to remote source when --checksums is enabled', async () => {
await expectExit(0, async () => {
await transferAction({
to: undefined,
from: sourceUrl,
fromToken: sourceToken,
checksums: true,
} as any);
});
expect(
mockDataTransfer.strapi.providers.createRemoteStrapiSourceProvider
).toHaveBeenCalledWith(
expect.objectContaining({
verifyChecksums: true,
})
);
});
it('does not pass verifyChecksums to remote source when checksums are disabled', async () => {
await expectExit(0, async () => {
await transferAction({
to: undefined,
from: sourceUrl,
fromToken: sourceToken,
checksums: false,
} as any);
});
expect(
mockDataTransfer.strapi.providers.createRemoteStrapiSourceProvider
).toHaveBeenCalledWith(
expect.not.objectContaining({
verifyChecksums: true,
})
);
});
it('passes server.transfer.remote.assetIdleTimeoutMs to remote source provider as streamTimeout', async () => {
(dataTransferUtils.createStrapiInstance as jest.Mock).mockImplementationOnce(async () => ({
config: {
get: (key: string) =>
key === 'server.transfer.remote.assetIdleTimeoutMs' ? 99_000 : undefined,
},
telemetry: {
send: jest.fn(),
},
contentTypes: {},
}));
await expectExit(0, async () => {
await transferAction({
to: undefined,
from: sourceUrl,
fromToken: sourceToken,
} as any);
});
expect(
mockDataTransfer.strapi.providers.createRemoteStrapiSourceProvider
).toHaveBeenCalledWith(
expect.objectContaining({
url: sourceUrl,
streamTimeout: 99_000,
})
);
});
it('uses local Strapi destination when to is not specified', async () => {
await expectExit(0, async () => {
await transferAction({
@@ -27,6 +27,17 @@ const {
},
} = strapiDataTransfer;
const resolveRemotePullAssetIdleTimeoutMs = (value: unknown): number | undefined => {
if (value == null || value === '') {
return undefined;
}
const n = typeof value === 'number' ? value : Number(value);
if (!Number.isFinite(n) || n <= 0) {
return undefined;
}
return n;
};
interface CmdOptions {
from?: URL;
fromToken: string;
@@ -37,6 +48,7 @@ interface CmdOptions {
exclude?: (keyof engineDataTransfer.TransferGroupFilter)[];
throttle?: number;
force?: boolean;
checksums?: boolean;
}
/**
* Transfer command.
@@ -54,6 +66,7 @@ export default async (opts: CmdOptions) => {
}
const strapi = await createStrapiInstance();
const checksumsEnabled = opts.checksums !== false;
let source;
let destination;
@@ -69,6 +82,10 @@ export default async (opts: CmdOptions) => {
exitWith(1, 'Missing token for remote destination');
}
const assetIdleTimeoutMs = resolveRemotePullAssetIdleTimeoutMs(
strapi.config.get('server.transfer.remote.assetIdleTimeoutMs')
);
source = createRemoteStrapiSourceProvider({
getStrapi: () => strapi,
url: opts.from,
@@ -76,6 +93,8 @@ export default async (opts: CmdOptions) => {
type: 'token',
token: opts.fromToken,
},
...(assetIdleTimeoutMs !== undefined ? { streamTimeout: assetIdleTimeoutMs } : {}),
...(checksumsEnabled ? { verifyChecksums: true } : {}),
});
}
@@ -101,6 +120,7 @@ export default async (opts: CmdOptions) => {
},
strategy: 'restore',
restore: parseRestoreFromOptions(opts, strapi),
...(checksumsEnabled ? { verifyChecksums: true } : {}),
});
}
@@ -145,6 +165,19 @@ export default async (opts: CmdOptions) => {
getAssetsBackupHandler(engine, { force: opts.force, action: 'transfer' })
);
// Update more frequently to ensure elapsed time is accurate even if the stage is not progressing
const activeStages = new Set<string>();
const lastStageData: Record<string, any> = {};
const interval = setInterval(() => {
for (const stage of activeStages) {
if (lastStageData[stage]) {
// Clone the lastStageData and ensure endTime is undefined so elapsed uses Date.now()
const dataCopy = { ...lastStageData[stage], endTime: undefined };
updateLoader(stage as any, { [stage]: dataCopy });
}
}
}, 100);
progress.on(`stage::start`, ({ stage, data }) => {
updateLoader(stage, data).start();
});
@@ -154,6 +187,8 @@ export default async (opts: CmdOptions) => {
});
progress.on('stage::progress', ({ stage, data }) => {
lastStageData[stage] = data[stage];
activeStages.add(stage);
updateLoader(stage, data);
});
@@ -161,6 +196,9 @@ export default async (opts: CmdOptions) => {
updateLoader(stage, data).fail();
});
progress.on('transfer::finish', () => clearInterval(interval));
progress.on('transfer::error', () => clearInterval(interval));
progress.on('transfer::start', async () => {
console.log(`Starting transfer...`);
@@ -36,6 +36,12 @@ const command = () => {
.addOption(
new Option('--to-token <token>', `Transfer token for the remote Strapi destination`)
)
.addOption(
new Option(
'--no-checksums',
'Disable end-to-end asset checksum verification for assets transfer'
)
)
.addOption(new Option('--verbose', 'Enable verbose logs'))
.addOption(forceOption)
.addOption(excludeOption)
@@ -0,0 +1,75 @@
import { loadersFactory } from '../data-transfer';
jest.mock('ora', () =>
jest.fn(() => ({
start: jest.fn(),
succeed: jest.fn(),
fail: jest.fn(),
text: '',
}))
);
describe('loadersFactory / updateLoader (transfer progress line)', () => {
test('includes count and size totals when totalBytes and totalCount are set', () => {
const { updateLoader, getLoader } = loadersFactory();
const t0 = 1_700_000_000_000;
const dateSpy = jest.spyOn(Date, 'now').mockReturnValue(t0 + 1000);
updateLoader('assets', {
assets: {
startTime: t0,
bytes: 100,
count: 1,
totalBytes: 400,
totalCount: 4,
},
});
const text = getLoader('assets').text;
expect(text).toContain('1 / 4');
expect(text).toMatch(/size:.*\/.*400/);
dateSpy.mockRestore();
});
test('includes eta when elapsed >= 500ms, bytes in range, and totalBytes known', () => {
const { updateLoader, getLoader } = loadersFactory();
const t0 = 1_700_000_000_000;
const dateSpy = jest.spyOn(Date, 'now').mockReturnValue(t0 + 1000);
updateLoader('assets', {
assets: {
startTime: t0,
bytes: 500,
count: 1,
totalBytes: 1500,
totalCount: 2,
},
});
expect(getLoader('assets').text).toContain('eta ~');
dateSpy.mockRestore();
});
test('omits eta when stage has finished (endTime set)', () => {
const { updateLoader, getLoader } = loadersFactory();
const t0 = 1_700_000_000_000;
const dateSpy = jest.spyOn(Date, 'now').mockReturnValue(t0 + 2000);
updateLoader('assets', {
assets: {
startTime: t0,
endTime: t0 + 2000,
bytes: 1500,
count: 2,
totalBytes: 1500,
totalCount: 2,
},
});
expect(getLoader('assets').text).not.toContain('eta ~');
dateSpy.mockRestore();
});
});
@@ -46,8 +46,8 @@ describe('logger', () => {
describe('formatDiagnostic', () => {
it('only creates a single logger', () => {
const info = true; // so we info reports are called
const diagnosticReporter = formatDiagnostic('export', info);
const verbose = true;
const diagnosticReporter = formatDiagnostic('export', verbose);
// Use the diagnostic reporter to log different levels of messages
diagnosticReporter({
@@ -83,29 +83,12 @@ describe('logger', () => {
expect(mockLogger.error).toHaveBeenCalledWith(expect.stringContaining('Test error'));
expect(mockLogger.warn).toHaveBeenCalledTimes(1);
expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('Test warning'));
expect(mockLogger.info).toHaveBeenCalledTimes(1);
expect(mockLogger.info).toHaveBeenCalledWith(expect.stringContaining('Test info'));
const infoMessages = mockLogger.info.mock.calls.map((c) => String(c[0]));
expect(infoMessages.some((m) => m.includes('Diagnostic log file'))).toBe(true);
expect(infoMessages.some((m) => m.includes('Test info'))).toBe(true);
});
it('does not log info if info is false', () => {
const diagnosticReporter = formatDiagnostic('export');
diagnosticReporter({
kind: 'info',
details: {
message: 'Test info',
createdAt: new Date(),
},
});
// Verify createLogger is not called
expect(createLogger).not.toBeCalled();
// Verify the mock logger does not receive logs
expect(mockLogger.info).not.toBeCalled();
});
it('does not log info if info is false, but logs error and warning logs', () => {
it('writes error, warning, and info to the diagnostic file when verbose (console) is false', () => {
const diagnosticReporter = formatDiagnostic('export');
// Use the diagnostic reporter to log different levels of messages
@@ -143,8 +126,8 @@ describe('logger', () => {
expect(mockLogger.warn).toHaveBeenCalledTimes(1);
expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('Test warning'));
// Verify the mock info logger does not receive logs
expect(mockLogger.info).not.toBeCalled();
const infoMessages = mockLogger.info.mock.calls.map((c) => String(c[0]));
expect(infoMessages.some((m) => m.includes('Test info'))).toBe(true);
});
});
});
@@ -1,4 +1,5 @@
import chalk from 'chalk';
import path from 'node:path';
import Table from 'cli-table3';
import { Command, Option } from 'commander';
import { configs, createLogger, type winston, formats } from '@strapi/logger';
@@ -8,7 +9,7 @@ import { merge } from 'lodash/fp';
import type { Core } from '@strapi/types';
import { engine as engineDataTransfer, strapi as strapiDataTransfer } from '@strapi/data-transfer';
import { readableBytes, exitWith } from './helpers';
import { readableBytes, readableTime, exitWith } from './helpers';
import { getParseListWithChoices, parseInteger, confirmMessage } from './commander';
const {
@@ -209,24 +210,36 @@ const errorColors = {
const formatDiagnostic = (
operation: string,
info?: boolean
verbose?: boolean
): Parameters<engineDataTransfer.TransferEngine['diagnostics']['onDiagnostic']>[0] => {
// Create log file for all incoming diagnostics
let logger: undefined | winston.Logger;
let logFileBasename: string | undefined;
const getLogger = () => {
if (!logger) {
logFileBasename = `${operation}_${Date.now()}.log`;
const absoluteLogPath = path.resolve(process.cwd(), logFileBasename);
logger = createLogger(
configs.createOutputFileConfiguration(`${operation}_${Date.now()}.log`, {
level: 'info',
format: formats?.detailedLogs,
})
configs.createOutputFileConfiguration(
logFileBasename,
{
level: 'info',
format: formats?.detailedLogs,
},
{
consoleLevel: verbose ? 'info' : 'warn',
}
)
);
logger.info(
`[${operation}] Diagnostic log file: ${absoluteLogPath} (info-level messages are written here even without --verbose)`
);
}
return logger;
};
// We don't want to write a log file until there is something to be logged
return ({ details, kind }) => {
try {
if (kind === 'error') {
@@ -237,7 +250,7 @@ const formatDiagnostic = (
getLogger().error(errorMessage);
}
if (kind === 'info' && info) {
if (kind === 'info') {
const { message, params, origin } = details;
const msg = `[${origin ?? 'transfer'}] ${message}\n${params ? JSON.stringify(params, null, 2) : ''}`;
@@ -265,6 +278,8 @@ type Data = {
endTime?: number;
bytes?: number;
count?: number;
totalBytes?: number;
totalCount?: number;
};
};
@@ -279,14 +294,40 @@ const loadersFactory = (defaultLoaders: Loaders = {} as Loaders) => {
const elapsedTime = stageData?.startTime
? (stageData?.endTime || Date.now()) - stageData.startTime
: 0;
const size = `size: ${readableBytes(stageData?.bytes ?? 0)}`;
const elapsed = `elapsed: ${elapsedTime} ms`;
const speed =
elapsedTime > 0 ? `(${readableBytes(((stageData?.bytes ?? 0) * 1000) / elapsedTime)}/s)` : '';
const bytes = stageData?.bytes ?? 0;
const count = stageData?.count ?? 0;
const totalBytes = stageData?.totalBytes;
const totalCount = stageData?.totalCount;
loaders[stage].text = `${stage}: ${stageData?.count ?? 0} transferred (${size}) (${elapsed}) ${
const countLabel =
totalCount != null && totalCount > 0 ? `${count} / ${totalCount}` : String(count);
const size =
totalBytes != null && totalBytes > 0
? `size: ${readableBytes(bytes)} / ${readableBytes(totalBytes)}`
: `size: ${readableBytes(bytes)}`;
const elapsed = `elapsed: ${readableTime(elapsedTime ?? 0)}`;
const speed = elapsedTime > 0 ? `(${readableBytes((bytes * 1000) / elapsedTime)}/s)` : '';
let eta = '';
if (
!stageData?.endTime &&
totalBytes != null &&
totalBytes > 0 &&
bytes < totalBytes &&
elapsedTime >= 500 &&
bytes > 0
) {
const speedBps = bytes / elapsedTime;
const remaining = totalBytes - bytes;
const etaMs = remaining / speedBps;
if (Number.isFinite(etaMs) && etaMs > 0 && etaMs < 86400000) {
eta = ` eta ~${readableTime(etaMs)}`;
}
}
loaders[stage].text = `${stage}: ${countLabel} transferred (${size}) (${elapsed}) ${
!stageData?.endTime ? speed : ''
}`;
}${eta}`;
return loaders[stage];
};
+31 -3
View File
@@ -19,13 +19,40 @@ const readableBytes = (bytes: number, decimals = 1, padStart = 0) => {
return '0';
}
const i = Math.floor(Math.log(bytes) / Math.log(bytesPerKb));
const result = `${parseFloat((bytes / bytesPerKb ** i).toFixed(decimals))} ${sizes[i].padStart(
2
)}`;
const result = `${(bytes / bytesPerKb ** i).toFixed(decimals)} ${sizes[i].padStart(2)}`;
return result.padStart(padStart);
};
// Helper to floor a number to a given number of decimal places
function floorToDecimal(value: number, decimals: number) {
const factor = 10 ** decimals;
return Math.floor(value * factor) / factor;
}
/**
* Convert elapsed time to a human readable formatted string, for example "1024" becomes "1s"
*/
const readableTime = (elapsedTime: number, decimals = 1, padStart = 0): string => {
let value: number;
let unit: string;
if (elapsedTime >= 60000) {
value = elapsedTime / 60000;
unit = 'm';
} else if (elapsedTime >= 1000) {
value = elapsedTime / 1000;
unit = 's';
} else {
value = elapsedTime;
unit = 'ms';
}
const floored = floorToDecimal(value, decimals);
const result = `${floored.toFixed(decimals)}${unit}`;
return result.padStart(padStart);
};
interface ExitWithOptions {
logger?: Console;
prc?: NodeJS.Process;
@@ -196,6 +223,7 @@ export {
assertUrlHasProtocol,
ifOptions,
readableBytes,
readableTime,
runAction,
assertCwdContainsStrapiProject,
notifyExperimentalCommand,
@@ -28,6 +28,12 @@ export interface ServerTransfer {
remote?:
| {
enabled?: boolean;
/**
* Max milliseconds without forward progress while pulling assets from a remote instance
* (`strapi transfer --from …`). Maps to the remote source provider stall timeout.
* Omit to use the package default (typically several minutes for large files over JSON/WS).
*/
assetIdleTimeoutMs?: number;
}
| undefined;
}
@@ -3,16 +3,26 @@ import { transports, LoggerOptions } from 'winston';
import { LEVEL_LABEL, LEVELS } from '../constants';
import { prettyPrint, excludeColors } from '../formats';
export type OutputFileLoggerOptions = {
/** Console only: omit noisy `info` when e.g. `warn` (file transport still uses `fileTransportOptions.level`). */
consoleLevel?: string;
};
export default (
filename: string,
fileTransportOptions: transports.FileTransportOptions = {}
fileTransportOptions: transports.FileTransportOptions = {},
options?: OutputFileLoggerOptions
): LoggerOptions => {
const consoleLevel = options?.consoleLevel ?? LEVEL_LABEL;
return {
level: LEVEL_LABEL,
levels: LEVELS,
format: prettyPrint(),
transports: [
new transports.Console(),
new transports.Console({
level: consoleLevel,
}),
new transports.File({
level: 'error',
filename,
@@ -18,7 +18,7 @@ const createTestTransferToken = async (strapi) => {
name: 'TestToken',
description: 'Transfer token used to seed the e2e database',
lifespan: null,
permissions: ['push'],
permissions: ['push', 'pull'],
accessKey: CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
});
}
+79 -1
View File
@@ -19,6 +19,84 @@ The `-c X` option can be used to limit the number of concurrently running domain
If any changes are made to the template, or other issues are being encountered, try removing and regenerating the test apps by using `yarn test:cli:clean` before running the tests.
### Passing Jest or Playwright CLI flags (`--testPathPattern`, etc.)
`tests/scripts/run-tests.js` parses **runner-only** options (`-d` / `--domains`, `-c` / `--concurrency`, `-f` / `--setup`) with yargs, then spawns **Jest** (CLI tests) or **Playwright** (e2e). Any other flags (for example `--testPathPattern`, `--grep`) are forwarded to that underlying tool.
- With **Yarn**: `yarn test:cli -d strapi --testPathPattern=pull-remote` (or `push-remote`) works.
- With **npm**, you must pass arguments through npm:
`npm run test:cli -- -d strapi --testPathPattern=pull-remote`
- You can still use `--` so everything after it is forwarded as raw argv, e.g.
`yarn test:cli -d strapi -- --testPathPattern pull-remote`
### Environment variables
Jest is started from `tests/utils/runners/cli-runner.js` via **execa** with a **small** `env` object: only what the runner must inject (`TEST_APPS`, `JWT_SECRET`). Execas default **`extendEnv: true`** merges that object **on top of** the parent `process.env`.
So you do **not** need to whitelist variables in the runner. For **stress runs**, set `TRANSFER_CLI_MEDIA_COUNT` / `TRANSFER_CLI_MEDIA_BYTES` in the same shell as `yarn test:cli` so the seed scripts see them (defaults are tiny: 2 files × 2048 bytes).
The same merge behavior applies to **e2e**: `tests/utils/runners/browser-runner.js` only sets `PORT`, `HOST`, `TEST_APP_PATH`, and `STRAPI_DISABLE_EE`; the rest of the environment is inherited from how you invoked `yarn test:e2e`.
### Remote transfer e2e (pull and push, generated media)
The `strapi` domain reserves **two** test apps (`tests/cli/tests/strapi/config.js`) for:
| Suite | File | Direction |
| ----- | --------------------------------------- | ------------------------- |
| Pull | `data-transfer/pull-remote.test.cli.js` | Remote → local (`--from`) |
| Push | `data-transfer/push-remote.test.cli.js` | Local → remote (`--to`) |
Both run on every `yarn test:cli -d strapi` with **small** synthetic media (defaults: `TRANSFER_CLI_MEDIA_COUNT=2`, `TRANSFER_CLI_MEDIA_BYTES=2048`). No env flag is required to run them.
Assertions go beyond row counts: each suite compares **Strapis stored content `hash` plus `size`** for every seeded upload row between source and destination (see `tests/utils/cli-transfer-remote-e2e/upload-db.js`), so corrupted or truncated assets should fail even if the file count stayed right.
**Stress testing only** — raise total payload by exporting larger values for the same variables before invoking the CLI tests:
- `TRANSFER_CLI_MEDIA_COUNT` — number of synthetic files (default `2`)
- `TRANSFER_CLI_MEDIA_BYTES` — bytes per file (default `2048`)
- `CLI_TRANSFER_REMOTE_PORT` — remote Strapi port (default `13710`; legacy alias `CLI_TRANSFER_PULL_REMOTE_PORT`)
Other optional overrides: `CLI_TRANSFER_REMOTE_JEST_TIMEOUT_MS`, `CLI_TRANSFER_REMOTE_RUNNER_TIMEOUT_MS` (legacy `CLI_TRANSFER_PULL_*`).
This flow is **Jest / `yarn test:cli`**, not Playwright. The IDE Playwright runner does not execute these suites.
**Pull** (seed on **remote**), default small run:
```bash
yarn test:cli -d strapi --testPathPattern=pull-remote
```
**Push** (seed on **local**), default small run:
```bash
yarn test:cli -d strapi --testPathPattern=push-remote
```
Stress — larger library (10 × 512KiB), **pull**:
```bash
TRANSFER_CLI_MEDIA_COUNT=10 TRANSFER_CLI_MEDIA_BYTES=524288 \
yarn test:cli -d strapi --testPathPattern=pull-remote
```
Same sizes, **push**:
```bash
TRANSFER_CLI_MEDIA_COUNT=10 TRANSFER_CLI_MEDIA_BYTES=524288 \
yarn test:cli -d strapi --testPathPattern=push-remote
```
Stress — ~2GiB total (20 × 100MiB; `104857600` = 100 × 1024² bytes):
```bash
TRANSFER_CLI_MEDIA_COUNT=20 TRANSFER_CLI_MEDIA_BYTES=104857600 \
yarn test:cli -d strapi --testPathPattern=pull-remote
```
Large seeds need **plenty of free disk** under `test-apps/cli`. The **strapi** CLI domain uses a **30minute** outer Jest budget by default (pull + push are heavy); it grows automatically when `TRANSFER_CLI_MEDIA_BYTES × TRANSFER_CLI_MEDIA_COUNT` exceeds 10MiB or 100MiB (up to **4 hours**). Other CLI domains keep a **2minute** outer budget.
Implementation for these tests lives in **`tests/utils/cli-transfer-remote-e2e/`** (single package: seeding, SQLite checks, HTTP wait, timeouts). The **`tests/utils/seed-cli-transfer-media.js`** file is only a thin CLI entry that forwards to that package.
## Writing tests
The [coffee](https://github.com/node-modules/coffee) library is used to run commands and expect input, complete prompts, etc. Please see their documentation for more details.
@@ -37,7 +115,7 @@ Your CLI commands being tested can then be run in that directory.
As the CLI generally does not require a running Strapi app, this is not managed by the CLI testing tool.
After tests for remote data-transfer are implemented, there will be utility functions available to assist in running one or more of the test apps in the background while other tests are run against it.
For remote pull/push transfer, the suites under `data-transfer/pull-remote.test.cli.js` and `push-remote.test.cli.js` start a Strapi process in the background on a fixed port; see **Remote transfer e2e** above.
### Structure
+2 -1
View File
@@ -2,6 +2,7 @@
module.exports = () => {
return {
testApps: 1,
// Second app is reserved for remote transfer CLI e2e (pull-remote / push-remote).
testApps: 2,
};
};
@@ -0,0 +1,124 @@
'use strict';
/**
* End-to-end CLI test: pull data (including upload files) from a running remote Strapi into a
* second local test app. Runs on every strapi CLI pass with tiny seeded media by default.
*
* Stress only — increase synthetic file count/size:
* TRANSFER_CLI_MEDIA_COUNT (default 2)
* TRANSFER_CLI_MEDIA_BYTES (default 2048)
*
* Helpers: tests/utils/cli-transfer-remote-e2e/
*/
const { spawn } = require('child_process');
const coffee = require('coffee');
const execa = require('execa');
const utils = require('../../../../utils');
const {
APP_TEMPLATE_CONSTANTS,
getRemotePort,
countUploadFiles,
getSeedUploadSignature,
jestSuiteTimeoutMs,
seedTransferTestMedia,
waitForHttpOk,
} = require('../../../../utils/cli-transfer-remote-e2e');
// eslint-disable-next-line import/extensions
const { resetDatabaseAndImportDataFromPathProgrammatic } = require('../../../../utils/dts-import');
const REMOTE_PORT = getRemotePort();
describe('strapi transfer pull — remote to local (generated media)', () => {
jest.setTimeout(jestSuiteTimeoutMs());
let remotePath;
let localPath;
let remoteChild;
const { CUSTOM_TRANSFER_TOKEN_ACCESS_KEY } = require(APP_TEMPLATE_CONSTANTS);
beforeAll(async () => {
const apps = utils.instances.getTestApps();
remotePath = apps[0];
localPath = apps[1];
if (!remotePath || !localPath) {
throw new Error('Expected TEST_APPS to list two app paths (strapi domain testApps: 2)');
}
await resetDatabaseAndImportDataFromPathProgrammatic(remotePath, 'with-admin');
await resetDatabaseAndImportDataFromPathProgrammatic(localPath, 'with-admin');
await seedTransferTestMedia(remotePath);
await execa('npm', ['run', '-s', 'build'], {
cwd: remotePath,
stdio: 'inherit',
env: { ...process.env, PATH: process.env.PATH },
});
const remoteFilesAfterSeed = countUploadFiles(remotePath);
expect(remoteFilesAfterSeed).toBeGreaterThan(0);
remoteChild = spawn('npm', ['run', '-s', 'start'], {
cwd: remotePath,
env: {
...process.env,
PORT: REMOTE_PORT,
HOST: '127.0.0.1',
},
stdio: 'ignore',
});
const base = `http://127.0.0.1:${REMOTE_PORT}`;
await waitForHttpOk(`${base}/admin`);
const resetTok = await fetch(`${base}/api/config/resettransfertoken`, { method: 'POST' });
if (!resetTok.ok) {
throw new Error(`resettransfertoken failed: ${resetTok.status}`);
}
});
afterAll(async () => {
if (remoteChild && !remoteChild.killed) {
remoteChild.kill('SIGTERM');
await new Promise((r) => setTimeout(r, 2000));
if (!remoteChild.killed) {
remoteChild.kill('SIGKILL');
}
}
});
it('pulls upload files from remote (counts + Strapi content hashes match)', async () => {
const remoteSig = getSeedUploadSignature(remotePath);
expect(remoteSig.files.length).toBeGreaterThan(0);
expect(getSeedUploadSignature(localPath).files).toHaveLength(0);
const remoteFiles = countUploadFiles(remotePath);
const fromUrl = `http://127.0.0.1:${REMOTE_PORT}/admin`;
await coffee
.spawn(
'npm',
[
'run',
'-s',
'strapi',
'--',
'transfer',
'--from',
fromUrl,
'--from-token',
CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
'--force',
],
{ cwd: localPath }
)
.expect('code', 0)
.end();
const localFiles = countUploadFiles(localPath);
expect(localFiles).toBe(remoteFiles);
expect(getSeedUploadSignature(localPath).files).toEqual(remoteSig.files);
});
});
@@ -0,0 +1,132 @@
'use strict';
/**
* End-to-end CLI test: push data (including upload files) from the local Strapi app to a running
* remote. Runs on every strapi CLI pass with tiny seeded media by default.
*
* Stress only — increase synthetic file count/size:
* TRANSFER_CLI_MEDIA_COUNT (default 2)
* TRANSFER_CLI_MEDIA_BYTES (default 2048)
*
* Remote port: CLI_TRANSFER_REMOTE_PORT (legacy: CLI_TRANSFER_PULL_REMOTE_PORT), default 13710
*
* Helpers: tests/utils/cli-transfer-remote-e2e/
*/
const { spawn } = require('child_process');
const coffee = require('coffee');
const execa = require('execa');
const utils = require('../../../../utils');
const {
APP_TEMPLATE_CONSTANTS,
getRemotePort,
countUploadFiles,
getSeedUploadSignature,
jestSuiteTimeoutMs,
seedTransferTestMedia,
waitForHttpOk,
} = require('../../../../utils/cli-transfer-remote-e2e');
// eslint-disable-next-line import/extensions
const { resetDatabaseAndImportDataFromPathProgrammatic } = require('../../../../utils/dts-import');
const REMOTE_PORT = getRemotePort();
describe('strapi transfer push — local to remote (generated media)', () => {
jest.setTimeout(jestSuiteTimeoutMs());
let remotePath;
let localPath;
let remoteChild;
const { CUSTOM_TRANSFER_TOKEN_ACCESS_KEY } = require(APP_TEMPLATE_CONSTANTS);
beforeAll(async () => {
const apps = utils.instances.getTestApps();
remotePath = apps[0];
localPath = apps[1];
if (!remotePath || !localPath) {
throw new Error('Expected TEST_APPS to list two app paths (strapi domain testApps: 2)');
}
await resetDatabaseAndImportDataFromPathProgrammatic(remotePath, 'with-admin');
await resetDatabaseAndImportDataFromPathProgrammatic(localPath, 'with-admin');
await seedTransferTestMedia(localPath);
await execa('npm', ['run', '-s', 'build'], {
cwd: localPath,
stdio: 'inherit',
env: { ...process.env, PATH: process.env.PATH },
});
await execa('npm', ['run', '-s', 'build'], {
cwd: remotePath,
stdio: 'inherit',
env: { ...process.env, PATH: process.env.PATH },
});
const localFilesAfterSeed = countUploadFiles(localPath);
expect(localFilesAfterSeed).toBeGreaterThan(0);
remoteChild = spawn('npm', ['run', '-s', 'start'], {
cwd: remotePath,
env: {
...process.env,
PORT: REMOTE_PORT,
HOST: '127.0.0.1',
},
stdio: 'ignore',
});
const base = `http://127.0.0.1:${REMOTE_PORT}`;
await waitForHttpOk(`${base}/admin`);
const resetTok = await fetch(`${base}/api/config/resettransfertoken`, { method: 'POST' });
if (!resetTok.ok) {
throw new Error(`resettransfertoken failed: ${resetTok.status}`);
}
});
afterAll(async () => {
if (remoteChild && !remoteChild.killed) {
remoteChild.kill('SIGTERM');
await new Promise((r) => setTimeout(r, 2000));
if (!remoteChild.killed) {
remoteChild.kill('SIGKILL');
}
}
});
it('pushes upload files to remote (counts + Strapi content hashes match)', async () => {
const localSig = getSeedUploadSignature(localPath);
expect(localSig.files.length).toBeGreaterThan(0);
expect(getSeedUploadSignature(remotePath).files).toHaveLength(0);
const localFiles = countUploadFiles(localPath);
const toUrl = `http://127.0.0.1:${REMOTE_PORT}/admin`;
await coffee
.spawn(
'npm',
[
'run',
'-s',
'strapi',
'--',
'transfer',
'--to',
toUrl,
'--to-token',
CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
'--force',
],
{ cwd: localPath }
)
.expect('code', 0)
.end();
const remoteFiles = countUploadFiles(remotePath);
expect(remoteFiles).toBe(localFiles);
expect(getSeedUploadSignature(remotePath).files).toEqual(localSig.files);
});
});
+6
View File
@@ -19,6 +19,12 @@ The `-c X` option can be used to limit the number of concurrent test apps, where
If any changes are made to the template, or other issues are being encountered, try removing and regenerating the test apps by using `yarn test:e2e:clean` before running the tests.
Options meant for Playwright (for example `--grep`) are forwarded by the unified runner after the runner-only flags (`-d`, `-c`, `-f`). With **npm**, pass them after `--`: `npm run test:e2e -- -d settings --grep "My test"`. See `tests/cli/README.md` for the same pattern on CLI/Jest.
### Environment variables
Playwright is spawned from `tests/utils/runners/browser-runner.js` with a **small** `env` override (`PORT`, `HOST`, `TEST_APP_PATH`, `STRAPI_DISABLE_EE`). **execa** merges that with the parent environment by default, so other variables you export before `yarn test:e2e` still reach Playwright and the app under test. Details and the parallel Jest behaviour are in `tests/cli/README.md`**Environment variables**.
## Additional Documentation
See contributor docs in `docs/docs/guides/e2e` for more detailed information about writing and maintaining e2e tests.
+6 -2
View File
@@ -16,6 +16,7 @@ const { runPlaywright } = require('../utils/runners/browser-runner');
const {
loadDomainConfigs,
calculateTestAppsRequired,
buildForwardedRunnerArgs,
runCLI,
} = require('../utils/runners/cli-runner');
const { createConfig } = require('../../playwright.base.config');
@@ -109,6 +110,8 @@ yargs
const { concurrency, domains: selectedDomains, setup, updateSnapshot } = testYargs;
const forwardedRunnerArgs = buildForwardedRunnerArgs(testYargs);
/**
* Publishing all packages to the yalc store
*/
@@ -396,7 +399,7 @@ module.exports = config
cwd,
port,
testAppPath,
testArgs: testYargs._,
testArgs: forwardedRunnerArgs,
});
})
);
@@ -438,7 +441,8 @@ module.exports = config
domainDir,
jestConfigPath,
testApps,
testArgs: [...(updateSnapshot ? ['-u'] : []), ...testYargs._],
testArgs: [...(updateSnapshot ? ['-u'] : []), ...forwardedRunnerArgs],
domain,
});
} catch (err) {
console.error('Test suite failed for', domain);
@@ -0,0 +1,14 @@
'use strict';
/**
* Upload filenames produced by seed-media.js; upload-db.js filters on the same prefix.
* Keep in sync: only this module defines the string.
*/
const SEED_UPLOAD_NAME_PREFIX = 'cli-pull-seed-';
/** SQLite LIKE pattern (includes wildcard). */
const SEED_UPLOAD_NAME_LIKE = `${SEED_UPLOAD_NAME_PREFIX}%`;
module.exports = {
SEED_UPLOAD_NAME_PREFIX,
SEED_UPLOAD_NAME_LIKE,
};
@@ -0,0 +1,31 @@
'use strict';
function getRemotePort() {
return (
process.env.CLI_TRANSFER_REMOTE_PORT || process.env.CLI_TRANSFER_PULL_REMOTE_PORT || '13710'
);
}
async function waitForHttpOk(url, { timeoutMs = 180000 } = {}) {
const start = Date.now();
// eslint-disable-next-line no-constant-condition
while (true) {
try {
const res = await fetch(url, { redirect: 'manual' });
if (res.status < 500) {
return;
}
} catch {
/* not up yet */
}
if (Date.now() - start > timeoutMs) {
throw new Error(`Timed out waiting for ${url}`);
}
await new Promise((r) => setTimeout(r, 400));
}
}
module.exports = {
getRemotePort,
waitForHttpOk,
};
@@ -0,0 +1,31 @@
'use strict';
/**
* Helpers for CLI remote transfer e2e (pull-remote / push-remote Jest suites).
*
* - {@link ./upload-db.js} SQLite file counts & seed-row signatures (in-process, no child scripts)
* - {@link ./seed-media.js} programmatic seed + CLI shim target
* - {@link ./timeouts.js} runner / Jest timeouts (also required directly from cli-runner)
* - {@link ./constants.js} shared seed filename prefix
*/
const path = require('path');
const { countUploadFiles, getSeedUploadSignature } = require('./upload-db');
const { seedTransferTestMedia } = require('./seed-media');
const { getRemotePort, waitForHttpOk } = require('./http');
const timeouts = require('./timeouts');
const REPO_ROOT = path.join(__dirname, '..', '..', '..');
const APP_TEMPLATE_CONSTANTS = path.join(REPO_ROOT, 'tests', 'app-template', 'src', 'constants.js');
module.exports = {
REPO_ROOT,
APP_TEMPLATE_CONSTANTS,
getRemotePort,
waitForHttpOk,
countUploadFiles,
getSeedUploadSignature,
seedTransferTestMedia,
...timeouts,
};
@@ -0,0 +1,99 @@
'use strict';
const fs = require('fs');
const path = require('path');
const os = require('os');
const { SEED_UPLOAD_NAME_PREFIX } = require('./constants');
/**
* Deterministic octet stream: every offset has a distinct value (per file index) so a
* full-file checksum catches chunk reordering; a single fill byte would not.
*/
function createDeterministicTransferTestFile(fileIndex, byteLength) {
const buf = Buffer.allocUnsafe(byteLength);
for (let j = 0; j < byteLength; j += 1) {
const mixed = (fileIndex + 1) * 0x9e3779b1 + j * 0x517cc1b7;
buf[j] = (mixed ^ (mixed >>> 11) ^ (j << 3)) & 255;
}
return buf;
}
function parseCountEnv() {
return Math.max(0, parseInt(process.env.TRANSFER_CLI_MEDIA_COUNT || '2', 10));
}
function parseBytesEnv() {
return Math.max(1, parseInt(process.env.TRANSFER_CLI_MEDIA_BYTES || '2048', 10));
}
/**
* @param {string} appPath - Strapi app root
* @param {{ count?: number, bytes?: number }} [options] - defaults from env TRANSFER_CLI_MEDIA_*
*/
async function seedTransferTestMedia(appPath, options = {}) {
const count = options.count ?? parseCountEnv();
const bytes = options.bytes ?? parseBytesEnv();
const { createStrapi } = require('@strapi/strapi');
const { CUSTOM_TRANSFER_TOKEN_ACCESS_KEY } = require(path.join(appPath, 'src', 'constants.js'));
const strapi = createStrapi({ appDir: appPath, distDir: appPath });
await strapi.load();
const { token: transferTokenService } = strapi.service('admin::transfer');
const existing = await transferTokenService.list();
for (const t of existing) {
await transferTokenService.revoke(t.id);
}
await transferTokenService.create({
name: 'CliTransferTestToken',
description: 'CLI remote transfer e2e',
lifespan: null,
permissions: ['push', 'pull'],
accessKey: CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
});
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'strapi-cli-transfer-seed-'));
try {
for (let i = 0; i < count; i += 1) {
const name = `${SEED_UPLOAD_NAME_PREFIX}${i}.bin`;
const tmpPath = path.join(tmpDir, name);
fs.writeFileSync(tmpPath, createDeterministicTransferTestFile(i, bytes));
await strapi
.plugin('upload')
.service('upload')
.upload({
data: {
fileInfo: { name },
},
files: [
{
filepath: tmpPath,
originalFilename: name,
mimetype: 'application/octet-stream',
size: bytes,
},
],
});
}
} finally {
await strapi.destroy();
fs.rmSync(tmpDir, { recursive: true, force: true });
}
return { count, bytes };
}
/** CLI entry: `node seed-cli-transfer-media.js [appPath]` */
async function runFromCli(argv) {
const appPath = argv[2] || process.cwd();
const { count, bytes } = await seedTransferTestMedia(appPath);
console.log(JSON.stringify({ ok: true, count, bytes }));
}
module.exports = {
seedTransferTestMedia,
runFromCli,
};
@@ -0,0 +1,68 @@
'use strict';
/**
* Jest / execa timeouts for remote transfer CLI e2e (scale with TRANSFER_CLI_MEDIA_*).
* Override: CLI_TRANSFER_REMOTE_*_TIMEOUT_MS (legacy CLI_TRANSFER_PULL_* for runner/jest).
*/
const parseNonNegativeInt = (value, fallback) => {
const n = parseInt(value ?? '', 10);
return Number.isFinite(n) && n >= 0 ? n : fallback;
};
const totalSeededMediaBytes = () => {
const bytes = parseNonNegativeInt(process.env.TRANSFER_CLI_MEDIA_BYTES, 2048);
const count = parseNonNegativeInt(process.env.TRANSFER_CLI_MEDIA_COUNT, 2);
return bytes * count;
};
const STRAPI_DOMAIN_DEFAULT_RUNNER_MS = 30 * 60 * 1000;
/**
* Outer execa(Jest) budget. Non-strapi CLI domains stay at 2 minutes.
* @param {string} [domainName]
*/
function runnerTimeoutMs(domainName) {
const explicit =
process.env.CLI_TRANSFER_REMOTE_RUNNER_TIMEOUT_MS ||
process.env.CLI_TRANSFER_PULL_RUNNER_TIMEOUT_MS;
if (explicit) {
return parseNonNegativeInt(explicit, STRAPI_DOMAIN_DEFAULT_RUNNER_MS);
}
if (domainName !== 'strapi') {
return 2 * 60 * 1000;
}
const total = totalSeededMediaBytes();
if (total > 100 * 1024 * 1024) {
return 4 * 60 * 60 * 1000;
}
if (total > 10 * 1024 * 1024) {
return 90 * 60 * 1000;
}
return STRAPI_DOMAIN_DEFAULT_RUNNER_MS;
}
function jestSuiteTimeoutMs() {
const explicit =
process.env.CLI_TRANSFER_REMOTE_JEST_TIMEOUT_MS ||
process.env.CLI_TRANSFER_PULL_JEST_TIMEOUT_MS;
if (explicit) {
return parseNonNegativeInt(explicit, 10 * 60 * 1000);
}
const total = totalSeededMediaBytes();
if (total > 100 * 1024 * 1024) {
return 4 * 60 * 60 * 1000;
}
if (total > 10 * 1024 * 1024) {
return 90 * 60 * 1000;
}
return 10 * 60 * 1000;
}
module.exports = {
totalSeededMediaBytes,
runnerTimeoutMs,
jestSuiteTimeoutMs,
};
@@ -0,0 +1,59 @@
'use strict';
const path = require('path');
const { SEED_UPLOAD_NAME_LIKE } = require('./constants');
function loadBetterSqlite3(appPath) {
try {
// Test apps ship the native module; resolve from app first for correct binary.
return require(path.join(appPath, 'node_modules', 'better-sqlite3'));
} catch {
return require('better-sqlite3');
}
}
function openUploadsDbReadonly(appPath) {
const Database = loadBetterSqlite3(appPath);
const dbPath = path.join(appPath, '.tmp', 'data.db');
return new Database(dbPath, { readonly: true });
}
function countUploadFiles(appPath) {
const db = openUploadsDbReadonly(appPath);
try {
const row = db.prepare('SELECT COUNT(*) AS c FROM files').get();
return row.c;
} finally {
db.close();
}
}
/**
* @returns {{ files: { name: string, hash: string, size: number }[] }}
*/
function getSeedUploadSignature(appPath) {
const db = openUploadsDbReadonly(appPath);
try {
const rows = db
.prepare(
`SELECT name, hash, size FROM files
WHERE name LIKE ?
ORDER BY name ASC`
)
.all(SEED_UPLOAD_NAME_LIKE);
return {
files: rows.map((r) => ({
name: r.name,
hash: r.hash,
size: Number(r.size),
})),
};
} finally {
db.close();
}
}
module.exports = {
countUploadFiles,
getSeedUploadSignature,
};
+3 -2
View File
@@ -3,14 +3,15 @@
const execa = require('execa');
/**
* Run Playwright test command
* Run Playwright test command.
* Only pass runner-owned env keys; execa merges with process.env by default (extendEnv: true).
*/
const runPlaywright = async ({ configPath, cwd, port, testAppPath, testArgs }) => {
await execa('yarn', ['playwright', 'test', '--config', configPath, ...testArgs], {
stdio: 'inherit',
cwd,
env: {
PORT: port,
PORT: String(port),
HOST: '127.0.0.1',
TEST_APP_PATH: testAppPath,
STRAPI_DISABLE_EE: !process.env.STRAPI_LICENSE,
+61 -8
View File
@@ -2,6 +2,7 @@
const path = require('path');
const execa = require('execa');
const { runnerTimeoutMs } = require('../cli-transfer-remote-e2e/timeouts');
/**
* Load domain-specific configuration
@@ -50,14 +51,59 @@ const calculateTestAppsRequired = (domainConfigs, concurrency) => {
};
/**
* Run Jest test command
* The test runner parses CLI flags with yargs before invoking Jest. Unknown options (e.g.
* `--testPathPattern=…`) become plain properties on the parsed object and are omitted from `_`,
* so they never reached Jest unless the user put them after `--`. Re-serialize those properties
* as argv fragments for Jest.
*
* @param {Record<string, unknown>} testYargs - yargs parse() result for args after `--type`
* @returns {string[]}
*/
const runCLI = async ({ domainDir, jestConfigPath, testApps, testArgs }) => {
const env = {
TEST_APPS: testApps.join(','),
JWT_SECRET: 'test-jwt-secret',
};
const buildForwardedRunnerArgs = (testYargs) => {
/** Keys owned by tests/scripts/run-tests.js (not for Jest). */
const runnerKeys = new Set([
'_',
'$0',
'concurrency',
'c',
'domains',
'd',
'setup',
'f',
'updateSnapshot',
'u',
]);
const args = [...testYargs._];
for (const key of Object.keys(testYargs)) {
if (runnerKeys.has(key) || key.startsWith('$')) {
continue;
}
const value = testYargs[key];
if (value === undefined || value === false) {
continue;
}
const flag = `--${key}`;
if (value === true) {
args.push(flag);
} else if (Array.isArray(value)) {
for (const item of value) {
args.push(flag, String(item));
}
} else {
args.push(`${flag}=${String(value)}`);
}
}
return args;
};
/**
* Run Jest test command
* @param {{ domainDir: string, jestConfigPath: string, testApps: string[], testArgs: string[], domain?: string }} opts
*/
const runCLI = async ({ domainDir, jestConfigPath, testApps, testArgs, domain }) => {
await execa(
'jest',
[
@@ -73,8 +119,14 @@ const runCLI = async ({ domainDir, jestConfigPath, testApps, testArgs }) => {
{
stdio: 'inherit',
cwd: domainDir, // run from the domain directory
env, // pass it our custom env values
timeout: 5 * 60 * 1000, // 5 minutes
// Only set what the runner owns; execa merges with process.env by default (extendEnv: true),
// so e.g. TRANSFER_CLI_MEDIA_* from the shell still reach Jest.
env: {
TEST_APPS: testApps.join(','),
JWT_SECRET: process.env.JWT_SECRET || 'test-jwt-secret',
},
// strapi domain includes remote transfer e2e; longer budget than other domains (see cli-transfer-remote-e2e/timeouts.js).
timeout: runnerTimeoutMs(domain),
}
);
};
@@ -82,5 +134,6 @@ const runCLI = async ({ domainDir, jestConfigPath, testApps, testArgs }) => {
module.exports = {
loadDomainConfigs,
calculateTestAppsRequired,
buildForwardedRunnerArgs,
runCLI,
};
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env node
'use strict';
/**
* CLI shim for programmatic seeding implementation lives in cli-transfer-remote-e2e/seed-media.js.
*
* node tests/utils/seed-cli-transfer-media.js [appPath]
*
* Env: TRANSFER_CLI_MEDIA_COUNT, TRANSFER_CLI_MEDIA_BYTES (see tests/cli/README.md)
*/
require('./cli-transfer-remote-e2e/seed-media')
.runFromCli(process.argv)
.catch((e) => {
console.error(e);
process.exit(1);
});