mirror of
https://github.com/strapi/strapi.git
synced 2026-05-03 16:22:30 +00:00
fix(data-transfer): fix large transfer crashes; show transfer progress (#23479)
* enhancement: update progress per chunk * fix: check stageprogress exists to make ts happy * chore: split progress tracker into two methods * test: fix lint * enhancement: display readable times * fix: speed indication for assets * fix: restore speed indicator and fix double counting * chore: clean up code * fix: data transfer memory leak * fix: yarn.lock * chore: fix memory logging * ci: complex project remote transfer * enhancement: optimized transfer mode * test(cli): data transfer and env passthrough * chore: only send one message warning of legacy mode * enhancement: show transfer totals and estimated time * test(cli): fix deterministic transfer test files * fix: push and pull shared utils backwards compatibility * fix(data-transfer): extract legacy asset chunk parsing and tighten transfer logging/test coverage * enhancement: checksum negotiation * enhancement: show skipped file warnings on client * fix: transfer diagnostics * test: fix open handle * fix: clear stall timeout for assets * chore: fix misleading comments and variables * test: fix misleading test * test: fix typo * test: make checks deterministic, less flaky * enhancement(data-transfer): speed up asset totals; widen assets start reply window on remote pull * fix(data-transfer): harden WebSocket JSON serialization for transfer frames * fix(data-transfer): more transfer hardening * test: fix test imports * fix: await async write * fix(data-transfer): resolve push transfer deadlock and harden async writes - Extract createAssetsDestinationWritable so Writable callbacks run before uploadStream completes (same WS batch as PassThrough chunks). - Add writable-async-write (write callback + drain/finished race; avoid hang on destroy). - Wire push/pull, remote-source, file & directory sources to shared write(). - Fire-and-forget pull flush: Promise.resolve(flush).catch(onError); guard missing stream inside try. - Add regression tests (assets writable, writable-async-write, handler checks). * fix(data-transfer): write push stream batches sequentially Use a for-loop with await write() instead of Promise.all over msg.data so non-asset stages respect one in-flight write per objectMode Writable and backpressure from writable-async-write. - Validate minChunksForBackpressure in assertReadStreamBackpressure - Add engine test for non-Buffer asset chunk byte progress (counts as 1) - Assert push.ts keeps sequential msg.data handling in static handler test * fix(data-transfer): align push streamAsset with remote-source and harden tests - Push handler: combine stream/end under one branch, error when start is missing or action is invalid; shorten stage write comments. - Engine version-matching tests: use a fresh createDestination() per engine so parallel transfers do not share destination writables (MaxListeners warnings). - File destination tests: mock createWriteStream with a new Writable per call. - CLI transfer tests: mock progress.stream so transfer::finish runs after transfer and clears the progress setInterval (fixes Jest worker hang). - Misc test cleanup: assets-destination timeout clearTimeout, collect listeners, writable-async-write teardown; tighten push/static test descriptions. * test: remove parity test * fix(data-transfer): harden collect() and stabilize transfer tests - collect(): settle once, remove listeners on resolve/reject, avoid double completion - engine tests: add expectHeapGrowthWithinNoise for heap smoke checks - CLI transfer tests: console spies in beforeAll; jest.restoreAllMocks in afterAll - stream test: remove removeAllListeners workaround
This commit is contained in:
@@ -4,6 +4,11 @@ const serverConfig = ({ env }) => ({
|
||||
app: {
|
||||
keys: env.array('APP_KEYS', ['toBeModified1', 'toBeModified2']),
|
||||
},
|
||||
transfer: {
|
||||
remote: {
|
||||
enabled: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
export default serverConfig;
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
"typescript": "^5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0 <=22.x.x",
|
||||
"node": ">=20.0.0 <=24.x.x",
|
||||
"npm": ">=6.0.0"
|
||||
},
|
||||
"strapi": {
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
"typescript": "^5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0 <=22.x.x",
|
||||
"node": ">=20.0.0 <=24.x.x",
|
||||
"npm": ">=6.0.0"
|
||||
},
|
||||
"strapi": {
|
||||
|
||||
@@ -1,7 +1,84 @@
|
||||
import { Readable } from 'stream';
|
||||
import { Readable, Writable } from 'stream';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import type { Core } from '@strapi/types';
|
||||
import type { ITransferEngine, ISourceProvider, IDestinationProvider } from '../../types';
|
||||
|
||||
/**
|
||||
* Create a slow Writable that applies backpressure (low highWaterMark, delayed write).
|
||||
* Used to verify that source streams pause when the consumer is slow.
|
||||
*/
|
||||
export const createSlowWritable = <T = unknown>(
|
||||
options: {
|
||||
objectMode?: boolean;
|
||||
highWaterMark?: number;
|
||||
delayMs?: number;
|
||||
onChunk?: (chunk: T) => void;
|
||||
} = {}
|
||||
): { writable: Writable; chunks: T[] } => {
|
||||
const { objectMode = true, highWaterMark = 1, delayMs = 10, onChunk } = options;
|
||||
const chunks: T[] = [];
|
||||
const writable = new Writable({
|
||||
objectMode,
|
||||
highWaterMark,
|
||||
write(chunk: T, _encoding, callback) {
|
||||
chunks.push(chunk);
|
||||
onChunk?.(chunk);
|
||||
setTimeout(callback, delayMs);
|
||||
},
|
||||
});
|
||||
return { writable, chunks };
|
||||
};
|
||||
|
||||
/**
|
||||
* Run a backpressure test on a Readable: pipe to a slow consumer and assert the source stream
|
||||
* was paused at least once (proving backpressure is applied). Returns collected chunks for integrity checks.
|
||||
*/
|
||||
export const assertReadStreamBackpressure = async <T = unknown>(
|
||||
stream: Readable,
|
||||
options: { delayMs?: number; minChunksForBackpressure?: number } = {}
|
||||
): Promise<{ sourcePaused: boolean; chunks: T[] }> => {
|
||||
const { delayMs = 10, minChunksForBackpressure } = options;
|
||||
let sourcePaused = false;
|
||||
const originalPause = stream.pause.bind(stream);
|
||||
stream.pause = function (this: Readable) {
|
||||
sourcePaused = true;
|
||||
return originalPause();
|
||||
};
|
||||
|
||||
const { writable, chunks } = createSlowWritable<T>({ delayMs, highWaterMark: 1 });
|
||||
await pipeline(stream, writable);
|
||||
|
||||
if (minChunksForBackpressure !== undefined && chunks.length < minChunksForBackpressure) {
|
||||
throw new Error(
|
||||
`assertReadStreamBackpressure: need at least ${minChunksForBackpressure} chunk(s) to meaningfully test backpressure, got ${chunks.length}`
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
sourcePaused,
|
||||
chunks,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Run a backpressure test on a Writable: pipe a fast Readable (many chunks) into it and assert
|
||||
* the readable was paused (proving the destination applies backpressure).
|
||||
*/
|
||||
export const assertWriteStreamBackpressure = async <T = unknown>(
|
||||
writable: Writable,
|
||||
chunks: T[]
|
||||
): Promise<{ sourcePaused: boolean }> => {
|
||||
const source = Readable.from(chunks, { objectMode: true });
|
||||
let sourcePaused = false;
|
||||
const originalPause = source.pause.bind(source);
|
||||
source.pause = function (this: Readable) {
|
||||
sourcePaused = true;
|
||||
return originalPause();
|
||||
};
|
||||
await pipeline(source, writable);
|
||||
return { sourcePaused };
|
||||
};
|
||||
|
||||
/**
|
||||
* Collect every entity in a Readable stream
|
||||
*/
|
||||
|
||||
@@ -4,6 +4,7 @@ import path from 'path';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
import { createLocalDirectorySourceProvider } from '..';
|
||||
import { assertReadStreamBackpressure } from '../../../../__tests__/test-utils';
|
||||
|
||||
const minimalMetadata = { strapi: { version: '5.0.0' }, createdAt: new Date().toISOString() };
|
||||
|
||||
@@ -58,6 +59,28 @@ describe('Directory source provider', () => {
|
||||
stream.destroy();
|
||||
});
|
||||
|
||||
test('entities read stream pauses under backpressure (slow consumer)', async () => {
|
||||
const dir = await fs.mkdtemp(path.join(tmpdir(), 'dts-dir-bp-'));
|
||||
await fs.writeJson(path.join(dir, 'metadata.json'), minimalMetadata);
|
||||
await fs.ensureDir(path.join(dir, 'entities'));
|
||||
const entityLines = Array.from(
|
||||
{ length: 25 },
|
||||
(_, i) => `${JSON.stringify({ type: 'api::test.test', id: i + 1, data: {} })}\n`
|
||||
).join('');
|
||||
await fs.writeFile(path.join(dir, 'entities', 'entities_00000.jsonl'), entityLines);
|
||||
|
||||
const provider = createLocalDirectorySourceProvider({ directory: { path: dir } });
|
||||
await provider.bootstrap({ report: jest.fn() } as never);
|
||||
|
||||
const stream = provider.createEntitiesReadStream();
|
||||
const { sourcePaused, chunks } = await assertReadStreamBackpressure(stream, {
|
||||
delayMs: 12,
|
||||
});
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
expect(chunks).toHaveLength(25);
|
||||
}, 8000);
|
||||
|
||||
test('streams entities from jsonl shards in order', async () => {
|
||||
const dir = await fs.mkdtemp(path.join(tmpdir(), 'dts-dir-'));
|
||||
await fs.writeJson(path.join(dir, 'metadata.json'), minimalMetadata);
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../
|
||||
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
|
||||
|
||||
import * as utils from '../../../utils';
|
||||
import { write } from '../../../utils/writable-async-write';
|
||||
import { ProviderInitializationError, ProviderTransferError } from '../../../errors/providers';
|
||||
import { unknownPathToPosix } from '../../../file/providers/source/utils';
|
||||
|
||||
@@ -196,7 +197,7 @@ class LocalDirectorySourceProvider implements ISourceProvider {
|
||||
stats: { size: stat.size },
|
||||
stream: fs.createReadStream(absUpload),
|
||||
};
|
||||
outStream.write(asset);
|
||||
await write(outStream, asset);
|
||||
}
|
||||
}
|
||||
outStream.end();
|
||||
@@ -246,7 +247,7 @@ class LocalDirectorySourceProvider implements ISourceProvider {
|
||||
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
outStream.write(chunk);
|
||||
await write(outStream, chunk);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
outStream.destroy(
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { posix, win32 } from 'path';
|
||||
import path, { posix, win32 } from 'path';
|
||||
import os from 'os';
|
||||
import fs from 'fs-extra';
|
||||
import { cloneDeep, get, set } from 'lodash/fp';
|
||||
import { Readable, Writable } from 'stream-chain';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import type { Struct } from '@strapi/types';
|
||||
import { createTransferEngine, TRANSFER_STAGES } from '..';
|
||||
|
||||
@@ -14,13 +17,19 @@ import type {
|
||||
ITransferEngineOptions,
|
||||
TransferFilterPreset,
|
||||
} from '../../../types';
|
||||
import {
|
||||
extendExpectForDataTransferTests,
|
||||
providerStages,
|
||||
sourceStages,
|
||||
} from '../../__tests__/test-utils';
|
||||
|
||||
import { extendExpectForDataTransferTests } from '../../__tests__/test-utils';
|
||||
import { TransferEngineValidationError } from '../errors';
|
||||
|
||||
/**
|
||||
* Parallel Jest + V8 can move `heapUsed` by tens of MB unrelated to transfer payload.
|
||||
* Use this as a loose smoke bound; structural tests (order, file bytes) are the real checks.
|
||||
*/
|
||||
function expectHeapGrowthWithinNoise(heapGrowth: number, totalBytes: number) {
|
||||
const limit = Math.max(totalBytes * 150, 32 * 1024 * 1024);
|
||||
expect(heapGrowth).toBeLessThan(limit);
|
||||
}
|
||||
|
||||
const getMockSourceStream = (data: Iterable<unknown>) => Readable.from(data);
|
||||
|
||||
const defaultLinksData: Array<ILink> = [
|
||||
@@ -690,20 +699,23 @@ describe('Transfer engine', () => {
|
||||
const source = createSource();
|
||||
const engine = createTransferEngine(source, completeDestination, defaultOptions);
|
||||
|
||||
let calls = 0;
|
||||
const progressEvents: Record<string, number> = {};
|
||||
engine.progress.stream.on('stage::progress', ({ stage, data }) => {
|
||||
expect(TRANSFER_STAGES.includes(stage)).toBe(true);
|
||||
expect(data).toMatchObject(engine.progress.data);
|
||||
calls += 1;
|
||||
progressEvents[stage] = (progressEvents[stage] || 0) + 1;
|
||||
});
|
||||
|
||||
await engine.transfer();
|
||||
|
||||
// Two values are emitted by default for each stage
|
||||
// TODO: this is no longer true, we should be checking the sum of the various mocked streams
|
||||
const itemPerStage = 3;
|
||||
// Each stage should emit at least one progress event
|
||||
TRANSFER_STAGES.forEach((stage) => {
|
||||
expect(progressEvents[stage]).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
expect(calls).toEqual((sourceStages.length - providerStages.length) * itemPerStage);
|
||||
// For assets, the number of progress events should match the number of chunks plus one for each asset's 'end' event
|
||||
// (from getAssetsMockSourceStream default: [ [1,2,3], [4,5,6,7,8,9] ] => 3 + 6 = 9, plus 2 'end' events = 11)
|
||||
expect(progressEvents.assets).toBe(11);
|
||||
});
|
||||
|
||||
test("emits 'stage::start' events", async () => {
|
||||
@@ -722,6 +734,31 @@ describe('Transfer engine', () => {
|
||||
expect(calls).toEqual(TRANSFER_STAGES.length);
|
||||
});
|
||||
|
||||
test('merges source getStageTotals into assets progress before stage::start', async () => {
|
||||
const source = createSource();
|
||||
source.getStageTotals = jest.fn().mockResolvedValue({ totalBytes: 12_345, totalCount: 7 });
|
||||
|
||||
const engine = createTransferEngine(source, completeDestination, defaultOptions);
|
||||
|
||||
let assetsAtStart: Record<string, unknown> | undefined;
|
||||
engine.progress.stream.on('stage::start', ({ stage, data }) => {
|
||||
if (stage === 'assets' && data.assets) {
|
||||
// Snapshot: `data` is the live progress object and mutates during the stage.
|
||||
assetsAtStart = { ...data.assets };
|
||||
}
|
||||
});
|
||||
|
||||
await engine.transfer();
|
||||
|
||||
expect(source.getStageTotals).toHaveBeenCalledWith('assets');
|
||||
expect(assetsAtStart).toMatchObject({
|
||||
totalBytes: 12_345,
|
||||
totalCount: 7,
|
||||
count: 0,
|
||||
bytes: 0,
|
||||
});
|
||||
});
|
||||
|
||||
test("emits 'stage::finish' events", async () => {
|
||||
const source = createSource();
|
||||
const engine = createTransferEngine(source, completeDestination, defaultOptions);
|
||||
@@ -902,7 +939,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -924,7 +961,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -937,7 +974,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -960,7 +997,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
await expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -975,7 +1012,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
await expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -999,7 +1036,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
await expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -1014,7 +1051,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
await expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -1038,7 +1075,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
await expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -1053,7 +1090,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
await expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -1076,7 +1113,7 @@ describe('Transfer engine', () => {
|
||||
modifiedMetadata.strapi.version = version;
|
||||
const source = createSource();
|
||||
source.getMetadata = jest.fn().mockResolvedValue(modifiedMetadata);
|
||||
const engine = createTransferEngine(source, completeDestination, options);
|
||||
const engine = createTransferEngine(source, createDestination(), options);
|
||||
await expect(
|
||||
(async () => {
|
||||
await engine.transfer();
|
||||
@@ -1120,7 +1157,7 @@ describe('Transfer engine', () => {
|
||||
createEntitiesReadStream() {
|
||||
const stream = Readable.from(sourceData, { objectMode: true });
|
||||
const originalPause = stream.pause.bind(stream);
|
||||
stream.pause = function () {
|
||||
stream.pause = function pause() {
|
||||
sourcePaused = true;
|
||||
return originalPause();
|
||||
};
|
||||
@@ -1202,7 +1239,7 @@ describe('Transfer engine', () => {
|
||||
createAssetsReadStream() {
|
||||
const stream = getAssetsMockSourceStream(assetData);
|
||||
const originalPause = stream.pause.bind(stream);
|
||||
stream.pause = function () {
|
||||
stream.pause = function pause() {
|
||||
sourcePaused = true;
|
||||
return originalPause();
|
||||
};
|
||||
@@ -1226,4 +1263,512 @@ describe('Transfer engine', () => {
|
||||
);
|
||||
}, 3000);
|
||||
});
|
||||
|
||||
describe('asset stream data and memory', () => {
|
||||
/**
|
||||
* Proves that asset stream bytes reach the destination. If the progress tracker
|
||||
* consumed the stream (old bug), the destination would receive 0 bytes per asset.
|
||||
*/
|
||||
test('destination receives full stream bytes for each asset (no double-consumption)', async () => {
|
||||
const expectedBytesPerAsset = [100, 250, 75];
|
||||
const assetData: IAsset[] = expectedBytesPerAsset.map((size, i) => ({
|
||||
filename: `file${i}.jpg`,
|
||||
filepath: posix.join(__dirname, `file${i}.jpg`),
|
||||
stats: { size },
|
||||
stream: Readable.from(Array.from({ length: size }, () => Buffer.alloc(1))),
|
||||
metadata: {
|
||||
hash: `hash${i}`,
|
||||
ext: '.jpg',
|
||||
id: i,
|
||||
name: '',
|
||||
mime: 'image/jpeg',
|
||||
size: 0,
|
||||
url: '',
|
||||
},
|
||||
}));
|
||||
|
||||
const bytesReceivedPerAsset: number[] = [];
|
||||
|
||||
const destination: IDestinationProvider = {
|
||||
...completeDestination,
|
||||
createAssetsWriteStream() {
|
||||
return new Writable({
|
||||
objectMode: true,
|
||||
async write(asset: IAsset, _encoding, callback) {
|
||||
if (!asset?.stream || typeof asset.stream.pipe !== 'function') {
|
||||
bytesReceivedPerAsset.push(0);
|
||||
return callback();
|
||||
}
|
||||
let bytes = 0;
|
||||
const counter = new Writable({
|
||||
write(chunk: Buffer | unknown, _enc, cb) {
|
||||
bytes += Buffer.isBuffer(chunk) ? chunk.length : 1;
|
||||
cb();
|
||||
},
|
||||
});
|
||||
try {
|
||||
await pipeline(asset.stream, counter);
|
||||
} catch (e) {
|
||||
return callback(e instanceof Error ? e : new Error(String(e)));
|
||||
}
|
||||
bytesReceivedPerAsset.push(bytes);
|
||||
callback();
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const source = createSource({ assets: assetData });
|
||||
const engine = createTransferEngine(source, destination, defaultOptions);
|
||||
await engine.transfer();
|
||||
|
||||
expect(bytesReceivedPerAsset).toEqual(expectedBytesPerAsset);
|
||||
}, 5000);
|
||||
|
||||
/**
|
||||
* Documents {@link TransferEngine}’s per-chunk progress rule: only `Buffer` chunks use
|
||||
* `.length`; anything else counts as 1 byte (cosmetic for ETA when chunk shapes are odd).
|
||||
*/
|
||||
test('progress byte totals: a non-Buffer chunk contributes 1 to the byte counter', async () => {
|
||||
const assetData: IAsset[] = [
|
||||
{
|
||||
filename: 'odd-chunk.bin',
|
||||
filepath: posix.join(__dirname, 'odd-chunk.bin'),
|
||||
stats: { size: 3 },
|
||||
stream: Readable.from([{ not: 'a buffer' }, Buffer.alloc(2)] as unknown[]),
|
||||
metadata: {
|
||||
hash: 'h2',
|
||||
ext: '.bin',
|
||||
id: 0,
|
||||
name: '',
|
||||
mime: 'application/octet-stream',
|
||||
size: 0,
|
||||
url: '',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const source = createSource({ assets: assetData });
|
||||
const engine = createTransferEngine(source, completeDestination, defaultOptions);
|
||||
await engine.transfer();
|
||||
|
||||
expect(engine.progress.data.assets?.bytes).toBe(3);
|
||||
}, 5000);
|
||||
|
||||
/**
|
||||
* Smoke check that we are not holding the whole payload many times over in heap.
|
||||
* (See {@link expectHeapGrowthWithinNoise} — absolute noise dominates small transfers.)
|
||||
*/
|
||||
test('heap growth during asset transfer stays bounded (streaming, not buffering)', async () => {
|
||||
const assetCount = 15;
|
||||
const bytesPerAsset = 50 * 1024; // 50KB each
|
||||
const totalBytes = assetCount * bytesPerAsset;
|
||||
const assetData: IAsset[] = Array.from({ length: assetCount }, (_, i) => ({
|
||||
filename: `large${i}.bin`,
|
||||
filepath: posix.join(__dirname, `large${i}.bin`),
|
||||
stats: { size: bytesPerAsset },
|
||||
stream: Readable.from(
|
||||
Array.from({ length: bytesPerAsset / 1024 }, () => Buffer.alloc(1024))
|
||||
),
|
||||
metadata: {
|
||||
hash: `h${i}`,
|
||||
ext: '.bin',
|
||||
id: i,
|
||||
name: '',
|
||||
mime: 'application/octet-stream',
|
||||
size: 0,
|
||||
url: '',
|
||||
},
|
||||
}));
|
||||
|
||||
const initialHeap = process.memoryUsage().heapUsed;
|
||||
const source = createSource({ assets: assetData });
|
||||
const engine = createTransferEngine(source, completeDestination, defaultOptions);
|
||||
await engine.transfer();
|
||||
const finalHeap = process.memoryUsage().heapUsed;
|
||||
const heapGrowth = finalHeap - initialHeap;
|
||||
|
||||
expectHeapGrowthWithinNoise(heapGrowth, totalBytes);
|
||||
}, 10000);
|
||||
});
|
||||
|
||||
describe('asset transfer integration (order, content, memory)', () => {
|
||||
/**
|
||||
* Full transfer path: source → engine (progress tracker) → destination that
|
||||
* "writes" each asset to a temp file. Verifies:
|
||||
* 1) Assets are received and written in correct order (no race).
|
||||
* 2) Each file's content matches the source (no mixing/corruption).
|
||||
* 3) Heap during transfer stays within a loose smoke bound (see {@link expectHeapGrowthWithinNoise}).
|
||||
*/
|
||||
test('full asset transfer: order preserved, content correct, memory bounded', async () => {
|
||||
const assetCount = 6;
|
||||
const bytesPerAsset = 10 * 1024; // 10KB each, unique byte value per asset
|
||||
const tmpDir = path.join(os.tmpdir(), `strapi-dt-integration-${Date.now()}`);
|
||||
await fs.ensureDir(tmpDir);
|
||||
|
||||
const assetData: IAsset[] = Array.from({ length: assetCount }, (_, i) => ({
|
||||
filename: `media-${i}.bin`,
|
||||
filepath: posix.join(__dirname, `media-${i}.bin`),
|
||||
stats: { size: bytesPerAsset },
|
||||
stream: Readable.from([Buffer.alloc(bytesPerAsset, i)]),
|
||||
metadata: {
|
||||
hash: `hash${i}`,
|
||||
ext: '.bin',
|
||||
id: i,
|
||||
name: `media-${i}`,
|
||||
mime: 'application/octet-stream',
|
||||
size: bytesPerAsset,
|
||||
url: '',
|
||||
},
|
||||
}));
|
||||
|
||||
const writeOrder: number[] = [];
|
||||
const memorySamples: number[] = [];
|
||||
let memoryInterval: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
const destination: IDestinationProvider = {
|
||||
...createDestination(),
|
||||
createAssetsWriteStream: jest.fn().mockResolvedValue(
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
async write(asset: IAsset, _encoding, callback) {
|
||||
const index = assetData.findIndex((a) => a.filename === asset.filename);
|
||||
writeOrder.push(index);
|
||||
const outPath = path.join(tmpDir, asset.filename);
|
||||
if (!asset?.stream || typeof asset.stream.pipe !== 'function') {
|
||||
return callback(new Error('Missing or invalid asset stream'));
|
||||
}
|
||||
try {
|
||||
await pipeline(asset.stream, fs.createWriteStream(outPath));
|
||||
} catch (e) {
|
||||
return callback(e instanceof Error ? e : new Error(String(e)));
|
||||
}
|
||||
callback();
|
||||
},
|
||||
})
|
||||
),
|
||||
};
|
||||
|
||||
const source = createSource({
|
||||
assets: assetData,
|
||||
schemas: Object.values(schemas) as Struct.Schema[],
|
||||
entities: [],
|
||||
links: [],
|
||||
configuration: [],
|
||||
});
|
||||
|
||||
memorySamples.push(process.memoryUsage().heapUsed);
|
||||
memoryInterval = setInterval(() => {
|
||||
memorySamples.push(process.memoryUsage().heapUsed);
|
||||
}, 25);
|
||||
|
||||
const engine = createTransferEngine(source, destination, {
|
||||
...defaultOptions,
|
||||
only: ['files'],
|
||||
});
|
||||
await engine.transfer();
|
||||
|
||||
if (memoryInterval) {
|
||||
clearInterval(memoryInterval);
|
||||
}
|
||||
memorySamples.push(process.memoryUsage().heapUsed);
|
||||
|
||||
const initialHeap = memorySamples[0];
|
||||
const maxHeap = Math.max(...memorySamples);
|
||||
const heapGrowth = maxHeap - initialHeap;
|
||||
const totalBytes = assetCount * bytesPerAsset;
|
||||
|
||||
expectHeapGrowthWithinNoise(heapGrowth, totalBytes);
|
||||
|
||||
expect(writeOrder).toHaveLength(assetCount);
|
||||
expect(writeOrder).toEqual([0, 1, 2, 3, 4, 5]);
|
||||
|
||||
for (let i = 0; i < assetCount; i += 1) {
|
||||
const content = await fs.readFile(path.join(tmpDir, `media-${i}.bin`));
|
||||
expect(content.length).toBe(bytesPerAsset);
|
||||
expect(content.every((b) => b === i)).toBe(true);
|
||||
}
|
||||
|
||||
await fs.remove(tmpDir).catch(() => {});
|
||||
}, 15000);
|
||||
});
|
||||
|
||||
describe('backpressure (schemas, links, configuration)', () => {
|
||||
test('schemas source stream pauses under backpressure and data integrity is maintained', async () => {
|
||||
const schemaData = getSchemasMockSourceStream();
|
||||
const schemaChunks = await new Promise<Struct.Schema[]>((resolve, reject) => {
|
||||
const chunks: Struct.Schema[] = [];
|
||||
schemaData.on('data', (chunk: Struct.Schema) => chunks.push(chunk));
|
||||
schemaData.on('end', () => resolve(chunks));
|
||||
schemaData.on('error', reject);
|
||||
});
|
||||
|
||||
let sourcePaused = false;
|
||||
const processedData: Struct.Schema[] = [];
|
||||
|
||||
const slowDestination: IDestinationProvider = {
|
||||
...completeDestination,
|
||||
createSchemasWriteStream() {
|
||||
return new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(chunk, _encoding, callback) {
|
||||
processedData.push(chunk);
|
||||
setTimeout(callback, 10);
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const source = {
|
||||
...createSource({ schemas: schemaChunks }),
|
||||
createSchemasReadStream() {
|
||||
const stream = getSchemasMockSourceStream(schemaChunks);
|
||||
const originalPause = stream.pause.bind(stream);
|
||||
stream.pause = function () {
|
||||
sourcePaused = true;
|
||||
return originalPause();
|
||||
};
|
||||
return stream;
|
||||
},
|
||||
};
|
||||
|
||||
const engine = createTransferEngine(source, slowDestination, defaultOptions);
|
||||
await engine.transfer();
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
expect(processedData).toEqual(schemaChunks);
|
||||
}, 2000);
|
||||
|
||||
test('links source stream pauses under backpressure and data integrity is maintained', async () => {
|
||||
const linksData = [...defaultLinksData];
|
||||
let sourcePaused = false;
|
||||
const processedData: ILink[] = [];
|
||||
|
||||
const slowDestination: IDestinationProvider = {
|
||||
...completeDestination,
|
||||
createLinksWriteStream() {
|
||||
return new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(chunk, _encoding, callback) {
|
||||
processedData.push(chunk);
|
||||
setTimeout(callback, 10);
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const source = {
|
||||
...createSource({ links: linksData }),
|
||||
createLinksReadStream() {
|
||||
const stream = getLinksMockSourceStream(linksData);
|
||||
const originalPause = stream.pause.bind(stream);
|
||||
stream.pause = function () {
|
||||
sourcePaused = true;
|
||||
return originalPause();
|
||||
};
|
||||
return stream;
|
||||
},
|
||||
};
|
||||
|
||||
const engine = createTransferEngine(source, slowDestination, defaultOptions);
|
||||
await engine.transfer();
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
expect(processedData).toHaveLength(linksData.length);
|
||||
expect(processedData).toEqual(expect.arrayContaining(linksData));
|
||||
}, 2000);
|
||||
|
||||
test('configuration source stream pauses under backpressure and data integrity is maintained', async () => {
|
||||
const configData: IConfiguration[] = [
|
||||
{ type: 'core-store', value: { key: 'foo', value: 'alice' } },
|
||||
{ type: 'core-store', value: { key: 'bar', value: 'bob' } },
|
||||
{ type: 'core-store', value: { key: 'baz', value: 'charlie' } },
|
||||
];
|
||||
let sourcePaused = false;
|
||||
const processedData: IConfiguration[] = [];
|
||||
|
||||
const slowDestination: IDestinationProvider = {
|
||||
...completeDestination,
|
||||
createConfigurationWriteStream() {
|
||||
return new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(chunk, _encoding, callback) {
|
||||
processedData.push(chunk);
|
||||
setTimeout(callback, 10);
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const source = {
|
||||
...createSource({ configuration: configData }),
|
||||
createConfigurationReadStream() {
|
||||
const stream = getConfigurationMockSourceStream(configData);
|
||||
const originalPause = stream.pause.bind(stream);
|
||||
stream.pause = function () {
|
||||
sourcePaused = true;
|
||||
return originalPause();
|
||||
};
|
||||
return stream;
|
||||
},
|
||||
};
|
||||
|
||||
const engine = createTransferEngine(source, slowDestination, defaultOptions);
|
||||
await engine.transfer();
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
expect(processedData).toEqual(configData);
|
||||
}, 2000);
|
||||
});
|
||||
|
||||
describe('stream cleanup and memory', () => {
|
||||
test('all stage streams are destroyed after successful transfer', async () => {
|
||||
const createdReadStreams: Readable[] = [];
|
||||
const createdWriteStreams: Writable[] = [];
|
||||
|
||||
const source = {
|
||||
...createSource(),
|
||||
createEntitiesReadStream: jest.fn().mockImplementation(async () => {
|
||||
const s = getEntitiesMockSourceStream();
|
||||
createdReadStreams.push(s);
|
||||
return s;
|
||||
}),
|
||||
createLinksReadStream: jest.fn().mockImplementation(() => {
|
||||
const s = getLinksMockSourceStream();
|
||||
createdReadStreams.push(s);
|
||||
return s;
|
||||
}),
|
||||
createAssetsReadStream: jest.fn().mockImplementation(async () => {
|
||||
const s = getAssetsMockSourceStream();
|
||||
createdReadStreams.push(s);
|
||||
return s;
|
||||
}),
|
||||
createConfigurationReadStream: jest.fn().mockImplementation(() => {
|
||||
const s = getConfigurationMockSourceStream();
|
||||
createdReadStreams.push(s);
|
||||
return s;
|
||||
}),
|
||||
createSchemasReadStream: jest.fn().mockImplementation(() => {
|
||||
const s = getSchemasMockSourceStream();
|
||||
createdReadStreams.push(s);
|
||||
return s;
|
||||
}),
|
||||
};
|
||||
|
||||
const destination = {
|
||||
...createDestination(),
|
||||
createEntitiesWriteStream: jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
createdWriteStreams.push(w);
|
||||
return w;
|
||||
}),
|
||||
createLinksWriteStream: jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
createdWriteStreams.push(w);
|
||||
return w;
|
||||
}),
|
||||
createAssetsWriteStream: jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
createdWriteStreams.push(w);
|
||||
return w;
|
||||
}),
|
||||
createConfigurationWriteStream: jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
createdWriteStreams.push(w);
|
||||
return w;
|
||||
}),
|
||||
createSchemasWriteStream: jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
createdWriteStreams.push(w);
|
||||
return w;
|
||||
}),
|
||||
};
|
||||
|
||||
const engine = createTransferEngine(source, destination, defaultOptions);
|
||||
await engine.transfer();
|
||||
|
||||
createdReadStreams.forEach((stream) => {
|
||||
expect(stream.destroyed).toBe(true);
|
||||
});
|
||||
createdWriteStreams.forEach((stream) => {
|
||||
expect(stream.destroyed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test('repeated transfers do not leave streams undestroyed', async () => {
|
||||
const transferCount = 5;
|
||||
const allReadStreams: Readable[] = [];
|
||||
const allWriteStreams: Writable[] = [];
|
||||
|
||||
const source = createSource();
|
||||
source.createEntitiesReadStream = jest.fn().mockImplementation(async () => {
|
||||
const s = getEntitiesMockSourceStream();
|
||||
allReadStreams.push(s);
|
||||
return s;
|
||||
});
|
||||
source.createLinksReadStream = jest.fn().mockImplementation(() => {
|
||||
const s = getLinksMockSourceStream();
|
||||
allReadStreams.push(s);
|
||||
return s;
|
||||
});
|
||||
source.createSchemasReadStream = jest.fn().mockImplementation(() => {
|
||||
const s = getSchemasMockSourceStream();
|
||||
allReadStreams.push(s);
|
||||
return s;
|
||||
});
|
||||
source.createAssetsReadStream = jest.fn().mockImplementation(async () => {
|
||||
const s = getAssetsMockSourceStream();
|
||||
allReadStreams.push(s);
|
||||
return s;
|
||||
});
|
||||
source.createConfigurationReadStream = jest.fn().mockImplementation(() => {
|
||||
const s = getConfigurationMockSourceStream();
|
||||
allReadStreams.push(s);
|
||||
return s;
|
||||
});
|
||||
|
||||
const destination = createDestination();
|
||||
destination.createEntitiesWriteStream = jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
allWriteStreams.push(w);
|
||||
return w;
|
||||
});
|
||||
destination.createLinksWriteStream = jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
allWriteStreams.push(w);
|
||||
return w;
|
||||
});
|
||||
destination.createSchemasWriteStream = jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
allWriteStreams.push(w);
|
||||
return w;
|
||||
});
|
||||
destination.createAssetsWriteStream = jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
allWriteStreams.push(w);
|
||||
return w;
|
||||
});
|
||||
destination.createConfigurationWriteStream = jest.fn().mockImplementation(async () => {
|
||||
const w = getMockDestinationStream();
|
||||
allWriteStreams.push(w);
|
||||
return w;
|
||||
});
|
||||
|
||||
const engine = createTransferEngine(source, destination, defaultOptions);
|
||||
|
||||
for (let i = 0; i < transferCount; i += 1) {
|
||||
await engine.transfer();
|
||||
}
|
||||
|
||||
allReadStreams.forEach((stream) => {
|
||||
expect(stream.destroyed).toBe(true);
|
||||
});
|
||||
allWriteStreams.forEach((stream) => {
|
||||
expect(stream.destroyed).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -32,6 +32,7 @@ import type {
|
||||
ErrorHandlerContext,
|
||||
ErrorHandlers,
|
||||
ErrorCode,
|
||||
StageProgress,
|
||||
} from '../../types';
|
||||
import type { Diff } from '../utils/json';
|
||||
|
||||
@@ -307,8 +308,7 @@ class TransferEngine<
|
||||
}
|
||||
|
||||
/**
|
||||
* Create and return a PassThrough stream.
|
||||
*
|
||||
* Create and return a PassThrough stream for per-object progress tracking.
|
||||
* Upon writing data into it, it'll update the Engine's transfer progress data and trigger stage update events.
|
||||
*/
|
||||
#progressTracker(
|
||||
@@ -328,6 +328,90 @@ class TransferEngine<
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Create and return a PassThrough stream for per-chunk progress tracking (used for assets).
|
||||
* Pipes each asset's binary stream through a Transform that counts bytes and forwards chunks,
|
||||
* then replaces asset.stream with that transform so the destination has a single consumer.
|
||||
* This avoids consuming the stream (which would leave the destination with an empty stream)
|
||||
* and ensures backpressure is applied so memory is not held for the entire transfer.
|
||||
*/
|
||||
#progressTrackerChunks(
|
||||
stage: TransferStage,
|
||||
aggregate?: {
|
||||
key?(value: unknown): string;
|
||||
}
|
||||
) {
|
||||
const updateAggregateBytes = this.#updateAggregateBytes.bind(this);
|
||||
const incrementAggregateCount = this.#incrementAggregateCount.bind(this);
|
||||
const emitStageUpdate = this.#emitStageUpdate.bind(this);
|
||||
|
||||
return new PassThrough({
|
||||
objectMode: true,
|
||||
transform: (asset, _encoding, callback) => {
|
||||
if (!asset?.stream || typeof asset.stream.pipe !== 'function') {
|
||||
return callback(null, asset);
|
||||
}
|
||||
|
||||
const key = aggregate?.key?.(asset);
|
||||
if (!this.progress.data[stage]) {
|
||||
this.progress.data[stage] = { count: 0, bytes: 0, startTime: Date.now() };
|
||||
}
|
||||
const stageProgress = this.progress.data[stage];
|
||||
|
||||
if (!stageProgress) {
|
||||
throw new TransferEngineError('fatal', 'Stage progress data not found');
|
||||
}
|
||||
|
||||
const progressTransform = new Transform({
|
||||
objectMode: true,
|
||||
transform(chunk: Buffer | unknown, _enc, cb) {
|
||||
// Asset file reads should yield Buffers; avoid skewing totals if not.
|
||||
const byteLength = Buffer.isBuffer(chunk) ? chunk.length : 1;
|
||||
stageProgress.bytes += byteLength;
|
||||
if (key) {
|
||||
updateAggregateBytes(stageProgress, key, byteLength);
|
||||
}
|
||||
emitStageUpdate('progress', stage);
|
||||
cb(null, chunk);
|
||||
},
|
||||
flush(cb) {
|
||||
stageProgress.count += 1;
|
||||
if (key) {
|
||||
incrementAggregateCount(stageProgress, key);
|
||||
}
|
||||
emitStageUpdate('progress', stage);
|
||||
cb(null);
|
||||
},
|
||||
});
|
||||
|
||||
asset.stream.on('error', (err: Error) => progressTransform.destroy(err));
|
||||
asset.stream.pipe(progressTransform);
|
||||
asset.stream = progressTransform;
|
||||
callback(null, asset);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
#updateAggregateBytes = (stageProgress: StageProgress, key: string, bytes: number) => {
|
||||
if (!stageProgress.aggregates) {
|
||||
stageProgress.aggregates = {};
|
||||
}
|
||||
if (!stageProgress.aggregates[key]) {
|
||||
stageProgress.aggregates[key] = { count: 0, bytes: 0 };
|
||||
}
|
||||
stageProgress.aggregates[key].bytes += bytes;
|
||||
};
|
||||
|
||||
#incrementAggregateCount = (stageProgress: StageProgress, key: string) => {
|
||||
if (!stageProgress.aggregates) {
|
||||
stageProgress.aggregates = {};
|
||||
}
|
||||
if (!stageProgress.aggregates[key]) {
|
||||
stageProgress.aggregates[key] = { count: 0, bytes: 0 };
|
||||
}
|
||||
stageProgress.aggregates[key].count += 1;
|
||||
};
|
||||
|
||||
/**
|
||||
* Shorthand method used to trigger transfer update events to every listeners
|
||||
*/
|
||||
@@ -589,6 +673,7 @@ class TransferEngine<
|
||||
if (!destination.destroyed) {
|
||||
destination.destroy(e as Error);
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
updateEndTime();
|
||||
}
|
||||
@@ -916,14 +1001,41 @@ class TransferEngine<
|
||||
const destination = await this.destinationProvider.createAssetsWriteStream?.();
|
||||
|
||||
const transform = this.#createStageTransformStream(stage);
|
||||
const tracker = this.#progressTracker(stage, {
|
||||
size: (value: IAsset) => value.stats.size,
|
||||
const tracker = this.#progressTrackerChunks(stage, {
|
||||
key: (value: IAsset) => extname(value.filename) || 'No extension',
|
||||
});
|
||||
|
||||
await this.#mergeSourceStageTotals(stage);
|
||||
await this.#transferStage({ stage, source, destination, transform, tracker });
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge optional source-reported totals into progress before the stage starts (CLI ETA / totals).
|
||||
*/
|
||||
async #mergeSourceStageTotals(stage: TransferStage) {
|
||||
const getTotals = this.sourceProvider.getStageTotals;
|
||||
if (!getTotals) {
|
||||
return;
|
||||
}
|
||||
const totals = await getTotals.call(this.sourceProvider, stage);
|
||||
if (!totals || (totals.totalBytes == null && totals.totalCount == null)) {
|
||||
return;
|
||||
}
|
||||
if (!this.progress.data[stage]) {
|
||||
this.progress.data[stage] = { count: 0, bytes: 0, startTime: Date.now() };
|
||||
}
|
||||
const stageProgress = this.progress.data[stage];
|
||||
if (!stageProgress) {
|
||||
return;
|
||||
}
|
||||
if (totals.totalBytes != null) {
|
||||
stageProgress.totalBytes = totals.totalBytes;
|
||||
}
|
||||
if (totals.totalCount != null) {
|
||||
stageProgress.totalCount = totals.totalCount;
|
||||
}
|
||||
}
|
||||
|
||||
async transferConfiguration(): Promise<void> {
|
||||
const stage: TransferStage = 'configuration';
|
||||
if (this.shouldSkipStage(stage)) {
|
||||
|
||||
@@ -6,14 +6,19 @@ jest.mock('fs');
|
||||
import fs from 'fs-extra';
|
||||
import { Writable } from 'stream-chain';
|
||||
import { createLocalFileDestinationProvider, ILocalFileDestinationProviderOptions } from '..';
|
||||
import {
|
||||
assertWriteStreamBackpressure,
|
||||
createSlowWritable,
|
||||
} from '../../../../__tests__/test-utils';
|
||||
import * as encryption from '../../../../utils/encryption';
|
||||
import { createFilePathFactory, createTarEntryStream } from '../utils';
|
||||
|
||||
fs.createWriteStream = jest.fn().mockReturnValue(
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
write() {},
|
||||
})
|
||||
fs.createWriteStream = jest.fn().mockImplementation(
|
||||
() =>
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
write() {},
|
||||
})
|
||||
);
|
||||
|
||||
const filePath = './test-file';
|
||||
@@ -225,4 +230,31 @@ describe('Local File Destination Provider', () => {
|
||||
expect(configurationStream instanceof stream.Writable).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Backpressure', () => {
|
||||
it('entities write stream applies backpressure when downstream is slow', async () => {
|
||||
const { writable: slowWritable } = createSlowWritable<string>({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
delayMs: 12,
|
||||
});
|
||||
(createTarEntryStream as jest.Mock).mockImplementation(() => slowWritable);
|
||||
|
||||
const provider = createLocalFileDestinationProvider({
|
||||
encryption: { enabled: false },
|
||||
compression: { enabled: false },
|
||||
file: { path: filePath },
|
||||
});
|
||||
await provider.bootstrap();
|
||||
|
||||
const writeStream = provider.createEntitiesWriteStream();
|
||||
const chunks = Array.from({ length: 25 }, (_, i) => ({ id: i, title: `Item ${i}` }));
|
||||
|
||||
const { sourcePaused } = await assertWriteStreamBackpressure(writeStream, chunks, {
|
||||
delayMs: 12,
|
||||
});
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
}, 5000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
import { Readable } from 'stream';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import fs from 'fs-extra';
|
||||
import tarStream from 'tar-stream';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import type { ILocalFileSourceProviderOptions } from '..';
|
||||
|
||||
import { createLocalFileSourceProvider } from '..';
|
||||
import { isFilePathInDirname, isPathEquivalent, unknownPathToPosix } from '../utils';
|
||||
import { assertReadStreamBackpressure } from '../../../../__tests__/test-utils';
|
||||
|
||||
describe('File source provider', () => {
|
||||
test('returns assets stream', () => {
|
||||
test('exposes createAssetsReadStream (starting the stream opens the backup file on disk)', () => {
|
||||
const options: ILocalFileSourceProviderOptions = {
|
||||
file: {
|
||||
path: './test-file',
|
||||
@@ -18,9 +23,7 @@ describe('File source provider', () => {
|
||||
},
|
||||
};
|
||||
const provider = createLocalFileSourceProvider(options);
|
||||
const stream = provider.createAssetsReadStream();
|
||||
|
||||
expect(stream instanceof Readable).toBeTruthy();
|
||||
expect(provider.createAssetsReadStream).toEqual(expect.any(Function));
|
||||
});
|
||||
|
||||
describe('utils', () => {
|
||||
@@ -130,4 +133,47 @@ describe('File source provider', () => {
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe('Backpressure', () => {
|
||||
test('entities read stream pauses under backpressure when reading from tar', async () => {
|
||||
const tmpDir = os.tmpdir();
|
||||
const tarPath = path.join(tmpDir, `strapi-dt-backpressure-${Date.now()}.tar`);
|
||||
const pack = tarStream.pack();
|
||||
|
||||
pack.entry(
|
||||
{ name: 'metadata.json' },
|
||||
JSON.stringify({
|
||||
createdAt: new Date().toISOString(),
|
||||
strapi: { version: '1.0.0' },
|
||||
})
|
||||
);
|
||||
const entityLines = Array.from(
|
||||
{ length: 25 },
|
||||
(_, i) => `${JSON.stringify({ uid: 'api::foo.foo', id: i + 1, title: `Entity ${i}` })}\n`
|
||||
).join('');
|
||||
pack.entry({ name: 'entities/entities_00000.jsonl' }, entityLines);
|
||||
pack.finalize();
|
||||
|
||||
const writeStream = fs.createWriteStream(tarPath);
|
||||
await pipeline(pack, writeStream);
|
||||
|
||||
const provider = createLocalFileSourceProvider({
|
||||
file: { path: tarPath },
|
||||
compression: { enabled: false },
|
||||
encryption: { enabled: false },
|
||||
});
|
||||
await provider.bootstrap();
|
||||
|
||||
const stream = provider.createEntitiesReadStream();
|
||||
const { sourcePaused, chunks } = await assertReadStreamBackpressure(stream, {
|
||||
delayMs: 12,
|
||||
minChunksForBackpressure: 10,
|
||||
});
|
||||
|
||||
await fs.remove(tarPath).catch(() => {});
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
expect(chunks.length).toBe(25);
|
||||
}, 8000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@ import type { IAsset, IMetadata, ISourceProvider, ProviderType, IFile } from '..
|
||||
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
|
||||
|
||||
import * as utils from '../../../utils';
|
||||
import { write } from '../../../utils/writable-async-write';
|
||||
import { ProviderInitializationError, ProviderTransferError } from '../../../errors/providers';
|
||||
import { isFilePathInDirname, isPathEquivalent, unknownPathToPosix } from './utils';
|
||||
|
||||
@@ -79,6 +80,29 @@ class LocalFileSourceProvider implements ISourceProvider {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* tar `Parser` invokes the pipeline completion callback when the archive ends, but it does not
|
||||
* reliably await async `onReadEntry` — defer `end` until outstanding async entry work is done.
|
||||
*/
|
||||
#endPassThroughWhenTarIdle(
|
||||
outStream: PassThrough,
|
||||
activeAsyncEntries: () => number,
|
||||
err?: Error | null
|
||||
) {
|
||||
if (err) {
|
||||
outStream.destroy(err);
|
||||
return;
|
||||
}
|
||||
const tick = () => {
|
||||
if (activeAsyncEntries() === 0) {
|
||||
outStream.end();
|
||||
} else {
|
||||
setImmediate(tick);
|
||||
}
|
||||
};
|
||||
tick();
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre flight checks regarding the provided options, making sure that the file can be opened (decrypted, decompressed), etc.
|
||||
*/
|
||||
@@ -167,6 +191,16 @@ class LocalFileSourceProvider implements ISourceProvider {
|
||||
const loadAssetMetadata = this.#loadAssetMetadata.bind(this);
|
||||
this.#reportInfo('creating assets read stream');
|
||||
|
||||
let activeAsyncEntries = 0;
|
||||
const runReadEntry = async (fn: () => Promise<void>) => {
|
||||
activeAsyncEntries += 1;
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
activeAsyncEntries -= 1;
|
||||
}
|
||||
};
|
||||
|
||||
pipeline(
|
||||
[
|
||||
inStream,
|
||||
@@ -179,27 +213,29 @@ class LocalFileSourceProvider implements ISourceProvider {
|
||||
return isFilePathInDirname('assets/uploads', filePath);
|
||||
},
|
||||
async onReadEntry(entry: ReadEntry) {
|
||||
const { path: filePath, size = 0 } = entry;
|
||||
const normalizedPath = unknownPathToPosix(filePath);
|
||||
const file = path.basename(normalizedPath);
|
||||
let metadata;
|
||||
try {
|
||||
metadata = await loadAssetMetadata(`assets/metadata/${file}.json`);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to read metadata for ${file}`);
|
||||
}
|
||||
const asset: IAsset = {
|
||||
metadata,
|
||||
filename: file,
|
||||
filepath: normalizedPath,
|
||||
stats: { size },
|
||||
stream: entry as unknown as Readable,
|
||||
};
|
||||
outStream.write(asset);
|
||||
await runReadEntry(async () => {
|
||||
const { path: filePath, size = 0 } = entry;
|
||||
const normalizedPath = unknownPathToPosix(filePath);
|
||||
const file = path.basename(normalizedPath);
|
||||
let metadata;
|
||||
try {
|
||||
metadata = await loadAssetMetadata(`assets/metadata/${file}.json`);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to read metadata for ${file}`);
|
||||
}
|
||||
const asset: IAsset = {
|
||||
metadata,
|
||||
filename: file,
|
||||
filepath: normalizedPath,
|
||||
stats: { size },
|
||||
stream: entry as unknown as Readable,
|
||||
};
|
||||
await write(outStream, asset);
|
||||
});
|
||||
},
|
||||
}),
|
||||
],
|
||||
() => outStream.end()
|
||||
(err) => this.#endPassThroughWhenTarIdle(outStream, () => activeAsyncEntries, err)
|
||||
);
|
||||
|
||||
return outStream;
|
||||
@@ -233,6 +269,16 @@ class LocalFileSourceProvider implements ISourceProvider {
|
||||
|
||||
const outStream = new PassThrough({ objectMode: true });
|
||||
|
||||
let activeAsyncEntries = 0;
|
||||
const runReadEntry = async (fn: () => Promise<void>) => {
|
||||
activeAsyncEntries += 1;
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
activeAsyncEntries -= 1;
|
||||
}
|
||||
};
|
||||
|
||||
pipeline(
|
||||
[
|
||||
inStream,
|
||||
@@ -246,43 +292,41 @@ class LocalFileSourceProvider implements ISourceProvider {
|
||||
},
|
||||
|
||||
async onReadEntry(entry: ReadEntry) {
|
||||
const transforms = [
|
||||
// JSONL parser to read the data chunks one by one (line by line)
|
||||
parser({
|
||||
checkErrors: true,
|
||||
}),
|
||||
// The JSONL parser returns each line as key/value
|
||||
(line: { key: string; value: object }) => line.value,
|
||||
];
|
||||
await runReadEntry(async () => {
|
||||
const transforms = [
|
||||
// JSONL parser to read the data chunks one by one (line by line)
|
||||
parser({
|
||||
checkErrors: true,
|
||||
}),
|
||||
// The JSONL parser returns each line as key/value
|
||||
(line: { key: string; value: object }) => line.value,
|
||||
];
|
||||
|
||||
const stream = entry.pipe(chain(transforms));
|
||||
const stream = entry.pipe(chain(transforms));
|
||||
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
outStream.write(chunk);
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
await write(outStream, chunk);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
outStream.destroy(
|
||||
new ProviderTransferError(
|
||||
`Error parsing backup files from backup file ${entry.path}: ${
|
||||
(e as Error).message
|
||||
}`,
|
||||
{
|
||||
details: {
|
||||
error: e,
|
||||
},
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
outStream.destroy(
|
||||
new ProviderTransferError(
|
||||
`Error parsing backup files from backup file ${entry.path}: ${
|
||||
(e as Error).message
|
||||
}`,
|
||||
{
|
||||
details: {
|
||||
error: e,
|
||||
},
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
}),
|
||||
],
|
||||
async () => {
|
||||
// Manually send the 'end' event to the out stream
|
||||
// once every entry has finished streaming its content
|
||||
outStream.end();
|
||||
}
|
||||
(err) => this.#endPassThroughWhenTarIdle(outStream, () => activeAsyncEntries, err)
|
||||
);
|
||||
|
||||
return outStream;
|
||||
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
import { Readable } from 'stream';
|
||||
import type { Core } from '@strapi/types';
|
||||
|
||||
import type { IAsset } from '../../../../../types';
|
||||
import { createTransaction } from '../../../../utils/transaction';
|
||||
import { createAssetsDestinationWritable } from '../assets-destination-writable';
|
||||
|
||||
/**
|
||||
* Push may send start + chunks + end in one batch; `write()` must return before `uploadStream`
|
||||
* resolves so the PassThrough is not read-only until upload completes (avoids assets-step deadlock).
|
||||
*/
|
||||
describe('createAssetsDestinationWritable (push transfer regression)', () => {
|
||||
test('invokes write callback before uploadStream completes (PassThrough can be fed in the same batch)', async () => {
|
||||
let releaseUpload!: (value?: unknown) => void;
|
||||
const uploadBlocked = new Promise((resolve) => {
|
||||
releaseUpload = resolve;
|
||||
});
|
||||
|
||||
let uploadFinished = false;
|
||||
const uploadStream = jest.fn(async () => {
|
||||
await uploadBlocked;
|
||||
uploadFinished = true;
|
||||
});
|
||||
|
||||
const mockFindOne = jest.fn().mockResolvedValue({ id: 1, url: 'x.jpg', formats: {} });
|
||||
const mockUpdate = jest.fn().mockResolvedValue(null);
|
||||
|
||||
const strapi = {
|
||||
config: {
|
||||
get(service: string) {
|
||||
if (service === 'plugin::upload') {
|
||||
return { provider: 'local' };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
},
|
||||
db: {
|
||||
transaction(fn: (arg: { trx: object; rollback: () => Promise<void> }) => Promise<void>) {
|
||||
fn({ trx: {}, rollback: async () => Promise.resolve() });
|
||||
return Promise.resolve();
|
||||
},
|
||||
query: jest.fn(() => ({
|
||||
findOne: mockFindOne,
|
||||
update: mockUpdate,
|
||||
})),
|
||||
},
|
||||
plugin(name: string) {
|
||||
if (name === 'upload') {
|
||||
return {
|
||||
provider: { uploadStream },
|
||||
};
|
||||
}
|
||||
return {};
|
||||
},
|
||||
} as unknown as Core.Strapi;
|
||||
|
||||
const transaction = createTransaction(strapi);
|
||||
|
||||
const stream = createAssetsDestinationWritable({
|
||||
strapi,
|
||||
transaction,
|
||||
resolveUploadFileId: () => 1,
|
||||
restoreMediaEntitiesContent: false,
|
||||
removeAssetsBackup: async () => Promise.resolve(),
|
||||
});
|
||||
|
||||
const file: IAsset = {
|
||||
filename: 'a.jpg',
|
||||
filepath: '/a',
|
||||
stats: { size: 10 },
|
||||
stream: Readable.from([Buffer.from('hello')]),
|
||||
metadata: {
|
||||
hash: 'h',
|
||||
name: 'a',
|
||||
id: 1,
|
||||
url: 'a.jpg',
|
||||
size: 10,
|
||||
mime: 'image/jpeg',
|
||||
},
|
||||
};
|
||||
|
||||
const writeSettled = new Promise<void>((resolve, reject) => {
|
||||
stream.write(file, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
writeSettled,
|
||||
new Promise<void>((_, reject) => {
|
||||
timeoutId = setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
new Error(
|
||||
'Timed out waiting for assets Writable write() callback (uploadStream must not block the write callback when start and chunks arrive in one batch)'
|
||||
)
|
||||
),
|
||||
200
|
||||
);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeoutId !== undefined) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
expect(uploadStream).toHaveBeenCalled();
|
||||
expect(uploadFinished).toBe(false);
|
||||
|
||||
releaseUpload();
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
setImmediate(resolve);
|
||||
});
|
||||
|
||||
expect(uploadFinished).toBe(true);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
stream.end((err?: Error | null) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
|
||||
transaction.end();
|
||||
});
|
||||
});
|
||||
+41
@@ -1,11 +1,14 @@
|
||||
import { createLocalStrapiDestinationProvider } from '../index';
|
||||
import * as restoreApi from '../strategies/restore';
|
||||
import {
|
||||
assertWriteStreamBackpressure,
|
||||
createSlowWritable,
|
||||
getStrapiFactory,
|
||||
getContentTypes,
|
||||
setGlobalStrapi,
|
||||
getStrapiModels,
|
||||
} from '../../../../__tests__/test-utils';
|
||||
import type { IEntity } from '../../../../../types';
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
@@ -253,4 +256,42 @@ describe('Local Strapi Source Destination', () => {
|
||||
expect(deleteAllSpy).toBeCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Backpressure', () => {
|
||||
test('entities write stream applies backpressure to slow down fast readables', async () => {
|
||||
const { writable: slowWritable } = createSlowWritable<IEntity>({
|
||||
highWaterMark: 1,
|
||||
delayMs: 15,
|
||||
});
|
||||
jest
|
||||
.spyOn(restoreApi, 'createEntitiesWriteStream')
|
||||
.mockReturnValue(slowWritable as ReturnType<typeof restoreApi.createEntitiesWriteStream>);
|
||||
|
||||
const provider = createLocalStrapiDestinationProvider({
|
||||
getStrapi: getStrapiFactory({
|
||||
db: {
|
||||
transaction,
|
||||
lifecycles: { enable: jest.fn(), disable: jest.fn() },
|
||||
},
|
||||
...strapiCommonProperties,
|
||||
}),
|
||||
strategy: 'restore',
|
||||
restore: { entities: { exclude: [] } },
|
||||
});
|
||||
await provider.bootstrap();
|
||||
|
||||
const writeStream = provider.createEntitiesWriteStream();
|
||||
const entityChunks: IEntity[] = Array.from({ length: 25 }, (_, i) => ({
|
||||
id: i + 1,
|
||||
type: 'api::foo.foo',
|
||||
data: { title: `Entity ${i}`, documentId: `doc-${i}` },
|
||||
}));
|
||||
|
||||
const { sourcePaused } = await assertWriteStreamBackpressure(writeStream, entityChunks, {
|
||||
delayMs: 15,
|
||||
});
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
}, 5000);
|
||||
});
|
||||
});
|
||||
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
import { Writable, Readable } from 'stream';
|
||||
import type { Core } from '@strapi/types';
|
||||
|
||||
import type { IAsset, IFile } from '../../../../types';
|
||||
import type { Transaction } from '../../../../types/utils';
|
||||
|
||||
export interface CreateAssetsDestinationWritableOptions {
|
||||
strapi: Core.Strapi;
|
||||
transaction: Transaction;
|
||||
resolveUploadFileId: (metadata: { id: number }) => number | undefined;
|
||||
restoreMediaEntitiesContent: boolean;
|
||||
removeAssetsBackup: () => Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writable for restoring upload assets during a local push destination transfer.
|
||||
*
|
||||
* The Writable `write()` callback must return as soon as the chunk is accepted — **before**
|
||||
* `uploadStream` finishes — so the remote push handler can feed PassThrough data in the same
|
||||
* WebSocket batch after an asset `start` row (see `streamAsset` in remote `push` handler).
|
||||
*/
|
||||
export function createAssetsDestinationWritable(
|
||||
options: CreateAssetsDestinationWritableOptions
|
||||
): Writable {
|
||||
const {
|
||||
strapi,
|
||||
transaction,
|
||||
resolveUploadFileId,
|
||||
restoreMediaEntitiesContent,
|
||||
removeAssetsBackup,
|
||||
} = options;
|
||||
|
||||
let pendingUploads = 0;
|
||||
|
||||
return new Writable({
|
||||
objectMode: true,
|
||||
async final(next) {
|
||||
while (pendingUploads > 0) {
|
||||
await new Promise<void>((resolve) => {
|
||||
setImmediate(resolve);
|
||||
});
|
||||
}
|
||||
await removeAssetsBackup();
|
||||
next();
|
||||
},
|
||||
write(chunk: IAsset, _encoding, callback) {
|
||||
const uploadData = {
|
||||
...chunk.metadata,
|
||||
stream: Readable.from(chunk.stream),
|
||||
buffer: chunk?.buffer,
|
||||
};
|
||||
|
||||
const provider = strapi.config.get<{ provider: string }>('plugin::upload').provider;
|
||||
|
||||
const fileId = resolveUploadFileId(uploadData);
|
||||
if (!fileId) {
|
||||
callback(new Error(`File ID not found for ID: ${uploadData.id}`));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!transaction) {
|
||||
callback(new Error('Transaction not available for asset upload'));
|
||||
return;
|
||||
}
|
||||
|
||||
pendingUploads += 1;
|
||||
transaction
|
||||
.attach(async () => {
|
||||
try {
|
||||
await strapi.plugin('upload').provider.uploadStream(uploadData);
|
||||
|
||||
if (!restoreMediaEntitiesContent) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (uploadData?.type) {
|
||||
const entry: IFile = await strapi.db.query('plugin::upload.file').findOne({
|
||||
where: { id: fileId },
|
||||
});
|
||||
if (!entry) {
|
||||
throw new Error('file not found');
|
||||
}
|
||||
const specificFormat = entry?.formats?.[uploadData.type];
|
||||
if (specificFormat) {
|
||||
specificFormat.url = uploadData.url;
|
||||
}
|
||||
await strapi.db.query('plugin::upload.file').update({
|
||||
where: { id: entry.id },
|
||||
data: {
|
||||
formats: entry.formats,
|
||||
provider,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: IFile = await strapi.db.query('plugin::upload.file').findOne({
|
||||
where: { id: fileId },
|
||||
});
|
||||
if (!entry) {
|
||||
throw new Error('file not found');
|
||||
}
|
||||
entry.url = uploadData.url;
|
||||
await strapi.db.query('plugin::upload.file').update({
|
||||
where: { id: entry.id },
|
||||
data: {
|
||||
url: entry.url,
|
||||
provider,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(`Error while uploading asset ${chunk.filename} ${error}`);
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
pendingUploads -= 1;
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
process.nextTick(() => {
|
||||
this.destroy(err);
|
||||
});
|
||||
});
|
||||
|
||||
callback();
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -3,16 +3,10 @@ import path from 'path';
|
||||
import * as fse from 'fs-extra';
|
||||
import type { Knex } from 'knex';
|
||||
import type { Core, Struct } from '@strapi/types';
|
||||
import type {
|
||||
IAsset,
|
||||
IDestinationProvider,
|
||||
IFile,
|
||||
IMetadata,
|
||||
ProviderType,
|
||||
Transaction,
|
||||
} from '../../../../types';
|
||||
import type { IDestinationProvider, IMetadata, ProviderType, Transaction } from '../../../../types';
|
||||
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
|
||||
|
||||
import { createAssetsDestinationWritable } from './assets-destination-writable';
|
||||
import { restore } from './strategies';
|
||||
import * as utils from '../../../utils';
|
||||
import {
|
||||
@@ -316,85 +310,14 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
|
||||
);
|
||||
}
|
||||
|
||||
const removeAssetsBackup = this.#removeAssetsBackup.bind(this);
|
||||
const strapi = this.strapi;
|
||||
const transaction = this.transaction;
|
||||
const fileEntitiesMapper = this.#entitiesMapper['plugin::upload.file'];
|
||||
|
||||
const restoreMediaEntitiesContent = this.#isContentTypeIncluded('plugin::upload.file');
|
||||
|
||||
return new Writable({
|
||||
objectMode: true,
|
||||
async final(next) {
|
||||
// Delete the backup folder
|
||||
await removeAssetsBackup();
|
||||
next();
|
||||
},
|
||||
async write(chunk: IAsset, _encoding, callback) {
|
||||
await transaction?.attach(async () => {
|
||||
const uploadData = {
|
||||
...chunk.metadata,
|
||||
stream: Readable.from(chunk.stream),
|
||||
buffer: chunk?.buffer,
|
||||
};
|
||||
|
||||
const provider = strapi.config.get<{ provider: string }>('plugin::upload').provider;
|
||||
|
||||
const fileId = fileEntitiesMapper?.[uploadData.id];
|
||||
if (!fileId) {
|
||||
return callback(new Error(`File ID not found for ID: ${uploadData.id}`));
|
||||
}
|
||||
|
||||
try {
|
||||
await strapi.plugin('upload').provider.uploadStream(uploadData);
|
||||
|
||||
// if we're not supposed to transfer the associated entities, stop here
|
||||
if (!restoreMediaEntitiesContent) {
|
||||
return callback();
|
||||
}
|
||||
|
||||
// Files formats are stored within the parent file entity
|
||||
if (uploadData?.type) {
|
||||
const entry: IFile = await strapi.db.query('plugin::upload.file').findOne({
|
||||
where: { id: fileId },
|
||||
});
|
||||
if (!entry) {
|
||||
throw new Error('file not found');
|
||||
}
|
||||
const specificFormat = entry?.formats?.[uploadData.type];
|
||||
if (specificFormat) {
|
||||
specificFormat.url = uploadData.url;
|
||||
}
|
||||
await strapi.db.query('plugin::upload.file').update({
|
||||
where: { id: entry.id },
|
||||
data: {
|
||||
formats: entry.formats,
|
||||
provider,
|
||||
},
|
||||
});
|
||||
return callback();
|
||||
}
|
||||
|
||||
const entry: IFile = await strapi.db.query('plugin::upload.file').findOne({
|
||||
where: { id: fileId },
|
||||
});
|
||||
if (!entry) {
|
||||
throw new Error('file not found');
|
||||
}
|
||||
entry.url = uploadData.url;
|
||||
await strapi.db.query('plugin::upload.file').update({
|
||||
where: { id: entry.id },
|
||||
data: {
|
||||
url: entry.url,
|
||||
provider,
|
||||
},
|
||||
});
|
||||
return callback();
|
||||
} catch (error) {
|
||||
return callback(new Error(`Error while uploading asset ${chunk.filename} ${error}`));
|
||||
}
|
||||
});
|
||||
},
|
||||
return createAssetsDestinationWritable({
|
||||
strapi: this.strapi,
|
||||
transaction: this.transaction!,
|
||||
resolveUploadFileId: (metadata) => fileEntitiesMapper?.[metadata.id],
|
||||
restoreMediaEntitiesContent: this.#isContentTypeIncluded('plugin::upload.file'),
|
||||
removeAssetsBackup: this.#removeAssetsBackup.bind(this),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
import { Readable } from 'stream';
|
||||
|
||||
import { createAssetsStream } from '../assets';
|
||||
|
||||
describe('Local source assets stream warnings', () => {
|
||||
test('reports warning callback when media DB row points to missing file', async () => {
|
||||
const missingFile = {
|
||||
id: 42,
|
||||
hash: 'missing-hash',
|
||||
ext: '.jpg',
|
||||
url: '/uploads/does-not-exist.jpg',
|
||||
provider: 'local',
|
||||
formats: undefined,
|
||||
};
|
||||
|
||||
const warn = jest.fn();
|
||||
const strapi = {
|
||||
db: {
|
||||
queryBuilder: jest.fn(() => ({
|
||||
select: jest.fn().mockReturnThis(),
|
||||
stream: jest.fn(() => Readable.from([missingFile])),
|
||||
})),
|
||||
},
|
||||
dirs: {
|
||||
static: { public: '/tmp/this-path-should-not-exist-for-test' },
|
||||
},
|
||||
log: {
|
||||
warn,
|
||||
},
|
||||
plugins: {
|
||||
upload: {
|
||||
provider: {
|
||||
isPrivate: jest.fn().mockResolvedValue(false),
|
||||
},
|
||||
},
|
||||
},
|
||||
config: {
|
||||
get: jest.fn(() => ({ provider: 'local' })),
|
||||
},
|
||||
} as any;
|
||||
|
||||
const onWarning = jest.fn();
|
||||
const stream = createAssetsStream(strapi, { onWarning });
|
||||
|
||||
const items: unknown[] = [];
|
||||
for await (const item of stream) {
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
expect(items).toHaveLength(0);
|
||||
expect(onWarning).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
'Media item 42 (hash: missing-hash) exists in database but no corresponding file was found to transfer'
|
||||
)
|
||||
);
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining('Media item 42'));
|
||||
});
|
||||
});
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
import { Readable } from 'stream';
|
||||
|
||||
import * as assets from '../assets';
|
||||
import { estimateAssetTotals } from '../estimate-asset-totals';
|
||||
|
||||
jest.mock('../assets', () => {
|
||||
const actual = jest.requireActual('../assets');
|
||||
return {
|
||||
__esModule: true,
|
||||
...actual,
|
||||
getFileStatsForTransfer: jest.fn(),
|
||||
signUploadFileForTransfer: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
});
|
||||
|
||||
describe('estimateAssetTotals', () => {
|
||||
const getStrapiMock = (rows: Record<string, unknown>[]) => ({
|
||||
dirs: { static: { public: '/data/public' } },
|
||||
log: { warn: jest.fn() },
|
||||
db: {
|
||||
queryBuilder: jest.fn(() => ({
|
||||
select: jest.fn().mockReturnThis(),
|
||||
stream: jest.fn(() => Readable.from(rows)),
|
||||
})),
|
||||
},
|
||||
plugins: {
|
||||
upload: {
|
||||
provider: {
|
||||
isPrivate: jest.fn().mockResolvedValue(false),
|
||||
},
|
||||
},
|
||||
},
|
||||
config: {
|
||||
get: jest.fn(() => ({ provider: 'local' })),
|
||||
},
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.mocked(assets.getFileStatsForTransfer).mockReset();
|
||||
jest.mocked(assets.signUploadFileForTransfer).mockClear();
|
||||
});
|
||||
|
||||
test('sums sizes and counts for local files (no formats)', async () => {
|
||||
jest.mocked(assets.getFileStatsForTransfer).mockResolvedValue({ size: 50 });
|
||||
const strapi = getStrapiMock([
|
||||
{
|
||||
provider: 'local',
|
||||
url: '/uploads/a.png',
|
||||
hash: 'h1',
|
||||
ext: '.png',
|
||||
id: 1,
|
||||
},
|
||||
{
|
||||
provider: 'local',
|
||||
url: '/uploads/b.png',
|
||||
hash: 'h2',
|
||||
ext: '.png',
|
||||
id: 2,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
|
||||
totalBytes: 100,
|
||||
totalCount: 2,
|
||||
});
|
||||
expect(assets.getFileStatsForTransfer).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test('includes format rows in totals', async () => {
|
||||
jest
|
||||
.mocked(assets.getFileStatsForTransfer)
|
||||
.mockResolvedValueOnce({ size: 100 })
|
||||
.mockResolvedValueOnce({ size: 30 });
|
||||
const strapi = getStrapiMock([
|
||||
{
|
||||
provider: 'local',
|
||||
url: '/uploads/main.jpg',
|
||||
hash: 'm',
|
||||
ext: '.jpg',
|
||||
id: 1,
|
||||
formats: {
|
||||
thumbnail: {
|
||||
url: '/uploads/thumb.jpg',
|
||||
hash: 't',
|
||||
ext: '.jpg',
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
|
||||
totalBytes: 130,
|
||||
totalCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
test('remote files use DB sizes only when main and formats have size (no HTTP stat)', async () => {
|
||||
const strapi = getStrapiMock([
|
||||
{
|
||||
provider: 'aws-s3',
|
||||
url: 'https://bucket/a.png',
|
||||
hash: 'h1',
|
||||
ext: '.png',
|
||||
id: 1,
|
||||
size: 400,
|
||||
formats: {
|
||||
thumbnail: {
|
||||
url: 'https://bucket/a_thumb.png',
|
||||
hash: 't',
|
||||
ext: '.png',
|
||||
size: 50,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
provider: 'aws-s3',
|
||||
url: 'https://bucket/b.png',
|
||||
hash: 'h2',
|
||||
ext: '.png',
|
||||
id: 2,
|
||||
size: 100,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
|
||||
totalBytes: 550,
|
||||
totalCount: 3,
|
||||
});
|
||||
|
||||
expect(assets.getFileStatsForTransfer).not.toHaveBeenCalled();
|
||||
expect(assets.signUploadFileForTransfer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('remote files fall back to HTTP stat when size is missing on main', async () => {
|
||||
jest.mocked(assets.getFileStatsForTransfer).mockResolvedValue({ size: 999 });
|
||||
const strapi = getStrapiMock([
|
||||
{
|
||||
provider: 'aws-s3',
|
||||
url: 'https://bucket/a.png',
|
||||
hash: 'h1',
|
||||
ext: '.png',
|
||||
id: 1,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
|
||||
totalBytes: 999,
|
||||
totalCount: 1,
|
||||
});
|
||||
|
||||
expect(assets.signUploadFileForTransfer).toHaveBeenCalledTimes(1);
|
||||
expect(assets.getFileStatsForTransfer).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('remote files fall back to HTTP stat when a format size is missing', async () => {
|
||||
jest.mocked(assets.getFileStatsForTransfer).mockResolvedValueOnce({ size: 12 }); // thumbnail probe only
|
||||
const strapi = getStrapiMock([
|
||||
{
|
||||
provider: 'aws-s3',
|
||||
url: 'https://bucket/a.png',
|
||||
hash: 'h1',
|
||||
ext: '.png',
|
||||
id: 1,
|
||||
size: 400,
|
||||
formats: {
|
||||
thumbnail: {
|
||||
url: 'https://bucket/a_thumb.png',
|
||||
hash: 't',
|
||||
ext: '.png',
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(estimateAssetTotals(strapi as any)).resolves.toEqual({
|
||||
totalBytes: 412,
|
||||
totalCount: 2,
|
||||
});
|
||||
|
||||
expect(assets.signUploadFileForTransfer).toHaveBeenCalledTimes(1);
|
||||
expect(assets.getFileStatsForTransfer).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { Readable } from 'stream';
|
||||
import type { IEntity } from '../../../../../types';
|
||||
|
||||
import {
|
||||
assertReadStreamBackpressure,
|
||||
collect,
|
||||
createMockedQueryBuilder,
|
||||
getStrapiFactory,
|
||||
@@ -258,4 +259,69 @@ describe('Local Strapi Source Provider', () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Backpressure', () => {
|
||||
test('entities read stream pauses under backpressure and preserves data', async () => {
|
||||
const itemCount = 25;
|
||||
const contentTypes = {
|
||||
foo: { uid: 'foo', attributes: { title: { type: 'string' } } },
|
||||
};
|
||||
const queryBuilder = createMockedQueryBuilder({
|
||||
foo: Array.from({ length: itemCount }, (_, i) => ({ id: i + 1, title: `Title ${i}` })),
|
||||
});
|
||||
|
||||
const provider = createLocalStrapiSourceProvider({
|
||||
getStrapi: getStrapiFactory({
|
||||
contentTypes,
|
||||
db: {
|
||||
queryBuilder,
|
||||
lifecycles: { enable: jest.fn(), disable: jest.fn() },
|
||||
},
|
||||
getModel: jest.fn((uid: string) => contentTypes[uid as keyof typeof contentTypes]),
|
||||
}),
|
||||
});
|
||||
await provider.bootstrap();
|
||||
|
||||
const stream = (await provider.createEntitiesReadStream()) as Readable;
|
||||
const { sourcePaused, chunks } = await assertReadStreamBackpressure<IEntity>(stream, {
|
||||
delayMs: 12,
|
||||
minChunksForBackpressure: 10,
|
||||
});
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
expect(chunks).toHaveLength(itemCount);
|
||||
chunks.forEach((entity) => {
|
||||
expect(entity).toMatchObject({
|
||||
type: 'foo',
|
||||
id: expect.any(Number),
|
||||
data: expect.any(Object),
|
||||
});
|
||||
});
|
||||
}, 5000);
|
||||
|
||||
test('schemas read stream pauses under backpressure', async () => {
|
||||
const contentTypes = {
|
||||
foo: { uid: 'foo', attributes: {} },
|
||||
bar: { uid: 'bar', attributes: {} },
|
||||
baz: { uid: 'baz', attributes: {} },
|
||||
};
|
||||
const components = {};
|
||||
const provider = createLocalStrapiSourceProvider({
|
||||
getStrapi: getStrapiFactory({
|
||||
contentTypes,
|
||||
components,
|
||||
db: { lifecycles: { enable: jest.fn(), disable: jest.fn() } },
|
||||
}),
|
||||
});
|
||||
await provider.bootstrap();
|
||||
|
||||
const stream = provider.createSchemasReadStream();
|
||||
const { sourcePaused, chunks } = await assertReadStreamBackpressure(stream, {
|
||||
delayMs: 12,
|
||||
});
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
expect(chunks).toHaveLength(3);
|
||||
}, 5000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -41,7 +41,7 @@ function getFileStream(
|
||||
return readableStream;
|
||||
}
|
||||
|
||||
function getFileStats(
|
||||
export function getFileStatsForTransfer(
|
||||
filepath: string,
|
||||
strapi: Core.Strapi,
|
||||
isLocal = false
|
||||
@@ -71,19 +71,17 @@ function getFileStats(
|
||||
});
|
||||
}
|
||||
|
||||
async function signFile(file: IFile) {
|
||||
export async function signUploadFileForTransfer(strapi: Core.Strapi, file: IFile) {
|
||||
const { provider } = strapi.plugins.upload;
|
||||
const { provider: providerName } = strapi.config.get('plugin.upload') as { provider: string };
|
||||
const isPrivate = await provider.isPrivate();
|
||||
if (file?.provider === providerName && isPrivate) {
|
||||
const signUrl = async (file: IFile) => {
|
||||
const signedUrl = await provider.getSignedUrl(file);
|
||||
file.url = signedUrl.url;
|
||||
const signUrl = async (f: IFile) => {
|
||||
const signedUrl = await provider.getSignedUrl(f);
|
||||
f.url = signedUrl.url;
|
||||
};
|
||||
|
||||
// Sign the original file
|
||||
await signUrl(file);
|
||||
// Sign each file format
|
||||
if (file.formats) {
|
||||
for (const format of Object.keys(file.formats)) {
|
||||
await signUrl(file.formats[format]);
|
||||
@@ -92,10 +90,23 @@ async function signFile(file: IFile) {
|
||||
}
|
||||
}
|
||||
|
||||
const missingAssetWarningMessage = (file: IFile, filepath: string, format?: string): string => {
|
||||
const formatPart = format ? ` (format: ${format})` : '';
|
||||
return `[Data transfer] Media item ${file.id} (hash: ${file.hash}) exists in database but no corresponding file was found to transfer${formatPart}. Path: ${filepath}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate and consume assets streams in order to stream each file individually
|
||||
*/
|
||||
export const createAssetsStream = (strapi: Core.Strapi): Duplex => {
|
||||
export const createAssetsStream = (
|
||||
strapi: Core.Strapi,
|
||||
options: { onWarning?: (message: string) => void } = {}
|
||||
): Duplex => {
|
||||
const warnMissingAsset = (message: string) => {
|
||||
strapi.log.warn(message);
|
||||
options.onWarning?.(message);
|
||||
};
|
||||
|
||||
const generator: () => AsyncGenerator<IAsset, void> = async function* () {
|
||||
const stream: Readable = strapi.db
|
||||
.queryBuilder('plugin::upload.file')
|
||||
@@ -108,10 +119,23 @@ export const createAssetsStream = (strapi: Core.Strapi): Duplex => {
|
||||
for await (const file of stream) {
|
||||
const isLocalProvider = file.provider === 'local';
|
||||
if (!isLocalProvider) {
|
||||
await signFile(file);
|
||||
await signUploadFileForTransfer(strapi, file);
|
||||
}
|
||||
const filepath = isLocalProvider ? join(strapi.dirs.static.public, file.url) : file.url;
|
||||
const stats = await getFileStats(filepath, strapi, isLocalProvider);
|
||||
let stats: { size: number };
|
||||
try {
|
||||
stats = await getFileStatsForTransfer(filepath, strapi, isLocalProvider);
|
||||
} catch (err: unknown) {
|
||||
const code =
|
||||
err && typeof err === 'object' && 'code' in err
|
||||
? (err as NodeJS.ErrnoException).code
|
||||
: undefined;
|
||||
if (code === 'ENOENT') {
|
||||
warnMissingAsset(missingAssetWarningMessage(file, filepath));
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const stream = getFileStream(filepath, strapi, isLocalProvider);
|
||||
|
||||
yield {
|
||||
@@ -128,7 +152,24 @@ export const createAssetsStream = (strapi: Core.Strapi): Duplex => {
|
||||
const fileFormatFilepath = isLocalProvider
|
||||
? join(strapi.dirs.static.public, fileFormat.url)
|
||||
: fileFormat.url;
|
||||
const fileFormatStats = await getFileStats(fileFormatFilepath, strapi, isLocalProvider);
|
||||
let fileFormatStats: { size: number };
|
||||
try {
|
||||
fileFormatStats = await getFileStatsForTransfer(
|
||||
fileFormatFilepath,
|
||||
strapi,
|
||||
isLocalProvider
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
const code =
|
||||
err && typeof err === 'object' && 'code' in err
|
||||
? (err as NodeJS.ErrnoException).code
|
||||
: undefined;
|
||||
if (code === 'ENOENT') {
|
||||
warnMissingAsset(missingAssetWarningMessage(file, fileFormatFilepath, format));
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const fileFormatStream = getFileStream(fileFormatFilepath, strapi, isLocalProvider);
|
||||
const metadata = { ...fileFormat, type: format, id: file.id, mainHash: file.hash };
|
||||
yield {
|
||||
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
import { join } from 'path';
|
||||
import type { Readable } from 'stream';
|
||||
import type { Core } from '@strapi/types';
|
||||
|
||||
import type { IAsset, StageTotalsEstimate } from '../../../../types';
|
||||
|
||||
import { getFileStatsForTransfer, signUploadFileForTransfer } from './assets';
|
||||
|
||||
type UploadFileRecord = IAsset['metadata'];
|
||||
|
||||
/** Strapi stores byte size on each file record; use for remote totals to avoid per-URL HTTP. */
|
||||
function hasReliableDbSize(size: unknown): size is number {
|
||||
return typeof size === 'number' && Number.isFinite(size) && size >= 0;
|
||||
}
|
||||
|
||||
/** When every main + format has a DB size, remote rows need no signing or HTTP stat. */
|
||||
function remoteRowCanUseDbOnly(file: UploadFileRecord): boolean {
|
||||
if (!hasReliableDbSize(file.size)) {
|
||||
return false;
|
||||
}
|
||||
if (!file.formats) {
|
||||
return true;
|
||||
}
|
||||
for (const key of Object.keys(file.formats)) {
|
||||
if (!hasReliableDbSize(file.formats[key].size)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sum sizes and counts for the same asset rows `createAssetsStream` would yield (main + formats),
|
||||
* skipping missing files with ENOENT like the stream does. Used for transfer progress totals / ETA.
|
||||
*
|
||||
* - **Local (`provider === 'local'`):** `stat` on disk (source of truth; matches ENOENT skips).
|
||||
* - **Remote:** sum `size` from DB when present on main and every format; otherwise sign + `fetch` / `Content-Length` like before.
|
||||
*/
|
||||
export async function estimateAssetTotals(strapi: Core.Strapi): Promise<StageTotalsEstimate> {
|
||||
let totalBytes = 0;
|
||||
let totalCount = 0;
|
||||
|
||||
const stream: Readable = strapi.db.queryBuilder('plugin::upload.file').select('*').stream();
|
||||
|
||||
for await (const file of stream) {
|
||||
const isLocalProvider = file.provider === 'local';
|
||||
|
||||
if (isLocalProvider) {
|
||||
const filepath = join(strapi.dirs.static.public, file.url);
|
||||
try {
|
||||
const stats = await getFileStatsForTransfer(filepath, strapi, true);
|
||||
totalBytes += stats.size;
|
||||
totalCount += 1;
|
||||
} catch (err: unknown) {
|
||||
const code =
|
||||
err && typeof err === 'object' && 'code' in err
|
||||
? (err as NodeJS.ErrnoException).code
|
||||
: undefined;
|
||||
if (code === 'ENOENT') {
|
||||
strapi.log.warn(`[Data transfer] Skipping missing asset file: ${filepath}`);
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (file.formats) {
|
||||
for (const format of Object.keys(file.formats)) {
|
||||
const fileFormat = file.formats[format];
|
||||
const fileFormatFilepath = join(strapi.dirs.static.public, fileFormat.url);
|
||||
try {
|
||||
const fileFormatStats = await getFileStatsForTransfer(fileFormatFilepath, strapi, true);
|
||||
totalBytes += fileFormatStats.size;
|
||||
totalCount += 1;
|
||||
} catch (err: unknown) {
|
||||
const code =
|
||||
err && typeof err === 'object' && 'code' in err
|
||||
? (err as NodeJS.ErrnoException).code
|
||||
: undefined;
|
||||
if (code === 'ENOENT') {
|
||||
strapi.log.warn(`[Data transfer] Skipping missing asset file: ${fileFormatFilepath}`);
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Remote: prefer DB sizes (fast); fall back to signed URL + HTTP where `size` is missing.
|
||||
if (remoteRowCanUseDbOnly(file)) {
|
||||
totalBytes += file.size;
|
||||
totalCount += 1;
|
||||
if (file.formats) {
|
||||
for (const format of Object.keys(file.formats)) {
|
||||
totalBytes += file.formats[format].size;
|
||||
totalCount += 1;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
await signUploadFileForTransfer(strapi, file);
|
||||
|
||||
if (hasReliableDbSize(file.size)) {
|
||||
totalBytes += file.size;
|
||||
totalCount += 1;
|
||||
} else {
|
||||
try {
|
||||
const stats = await getFileStatsForTransfer(file.url, strapi, false);
|
||||
totalBytes += stats.size;
|
||||
totalCount += 1;
|
||||
} catch (err: unknown) {
|
||||
const code =
|
||||
err && typeof err === 'object' && 'code' in err
|
||||
? (err as NodeJS.ErrnoException).code
|
||||
: undefined;
|
||||
if (code === 'ENOENT') {
|
||||
strapi.log.warn(`[Data transfer] Skipping missing asset file: ${file.url}`);
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
if (file.formats) {
|
||||
for (const format of Object.keys(file.formats)) {
|
||||
const fileFormat = file.formats[format];
|
||||
const fileFormatFilepath = fileFormat.url;
|
||||
|
||||
if (hasReliableDbSize(fileFormat.size)) {
|
||||
totalBytes += fileFormat.size;
|
||||
totalCount += 1;
|
||||
} else {
|
||||
try {
|
||||
const fileFormatStats = await getFileStatsForTransfer(
|
||||
fileFormatFilepath,
|
||||
strapi,
|
||||
false
|
||||
);
|
||||
totalBytes += fileFormatStats.size;
|
||||
totalCount += 1;
|
||||
} catch (err: unknown) {
|
||||
const code =
|
||||
err && typeof err === 'object' && 'code' in err
|
||||
? (err as NodeJS.ErrnoException).code
|
||||
: undefined;
|
||||
if (code === 'ENOENT') {
|
||||
strapi.log.warn(`[Data transfer] Skipping missing asset file: ${fileFormatFilepath}`);
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { totalBytes, totalCount };
|
||||
}
|
||||
@@ -2,12 +2,13 @@ import { Readable } from 'stream';
|
||||
import { chain } from 'stream-chain';
|
||||
import type { Core, Struct } from '@strapi/types';
|
||||
|
||||
import type { IMetadata, ISourceProvider, ProviderType } from '../../../../types';
|
||||
import type { IMetadata, ISourceProvider, ProviderType, TransferStage } from '../../../../types';
|
||||
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
|
||||
import { createEntitiesStream, createEntitiesTransformStream } from './entities';
|
||||
import { createLinksStream } from './links';
|
||||
import { createConfigurationStream } from './configuration';
|
||||
import { createAssetsStream } from './assets';
|
||||
import { estimateAssetTotals } from './estimate-asset-totals';
|
||||
import * as utils from '../../../utils';
|
||||
import { assertValidStrapi } from '../../../utils/providers';
|
||||
|
||||
@@ -53,6 +54,17 @@ class LocalStrapiSourceProvider implements ISourceProvider {
|
||||
});
|
||||
}
|
||||
|
||||
#reportWarning(message: string) {
|
||||
this.#diagnostics?.report({
|
||||
details: {
|
||||
createdAt: new Date(),
|
||||
message,
|
||||
origin: 'local-source-provider',
|
||||
},
|
||||
kind: 'warning',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports an error to the diagnostic reporter.
|
||||
*/
|
||||
@@ -97,7 +109,8 @@ class LocalStrapiSourceProvider implements ISourceProvider {
|
||||
|
||||
getMetadata(): IMetadata {
|
||||
this.#reportInfo('getting metadata');
|
||||
const strapiVersion = strapi.config.get<string>('info.strapi');
|
||||
assertValidStrapi(this.strapi);
|
||||
const strapiVersion = this.strapi.config.get<string>('info.strapi');
|
||||
const createdAt = new Date().toISOString();
|
||||
|
||||
return {
|
||||
@@ -152,13 +165,25 @@ class LocalStrapiSourceProvider implements ISourceProvider {
|
||||
assertValidStrapi(this.strapi, 'Not able to stream assets');
|
||||
this.#reportInfo('creating assets read stream');
|
||||
|
||||
const stream = createAssetsStream(this.strapi);
|
||||
const stream = createAssetsStream(this.strapi, {
|
||||
onWarning: (message) => this.#reportWarning(message),
|
||||
});
|
||||
stream.on('error', (err) => {
|
||||
this.#handleStreamError('assets', err);
|
||||
});
|
||||
|
||||
return stream;
|
||||
}
|
||||
|
||||
async getStageTotals(stage: TransferStage) {
|
||||
if (stage !== 'assets') {
|
||||
return null;
|
||||
}
|
||||
assertValidStrapi(this.strapi, 'Not able to estimate asset totals');
|
||||
return estimateAssetTotals(this.strapi);
|
||||
}
|
||||
}
|
||||
|
||||
export type ILocalStrapiSourceProvider = InstanceType<typeof LocalStrapiSourceProvider>;
|
||||
|
||||
export { estimateAssetTotals } from './estimate-asset-totals';
|
||||
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
import type { IAsset } from '../../../../../types';
|
||||
import { createRemoteStrapiDestinationProvider } from '..';
|
||||
|
||||
import { connectToWebsocket, createDispatcher } from '../../utils';
|
||||
|
||||
jest.mock('../../utils', () => ({
|
||||
...jest.requireActual('../../utils'),
|
||||
connectToWebsocket: jest.fn(),
|
||||
createDispatcher: jest.fn(),
|
||||
}));
|
||||
|
||||
class MockWebSocket extends EventEmitter {
|
||||
send = jest.fn((_payload: string, cb?: (err?: Error) => void) => cb?.());
|
||||
|
||||
close = jest.fn();
|
||||
}
|
||||
|
||||
describe('Remote destination checksum negotiation', () => {
|
||||
test('warns and continues without checksums when peer does not support checksum negotiation', async () => {
|
||||
const ws = new MockWebSocket();
|
||||
(connectToWebsocket as jest.Mock).mockResolvedValue(ws);
|
||||
|
||||
const streamBatches: unknown[][] = [];
|
||||
const dispatcher = {
|
||||
dispatchCommand: jest.fn().mockResolvedValue({ transferID: 't1' }), // no `checksums` support
|
||||
dispatchTransferAction: jest.fn().mockResolvedValue(null),
|
||||
dispatchTransferStep: jest.fn(
|
||||
async (msg: { action: string; data?: unknown; step?: string }) => {
|
||||
if (msg.action === 'stream' && msg.step === 'assets' && Array.isArray(msg.data)) {
|
||||
streamBatches.push(msg.data);
|
||||
}
|
||||
if (msg.action === 'start') {
|
||||
return { ok: true };
|
||||
}
|
||||
if (msg.action === 'end') {
|
||||
return { ok: true, stats: { started: 1, finished: 1 } };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
),
|
||||
setTransferProperties: jest.fn(),
|
||||
dispatch: jest.fn(),
|
||||
transferID: 't1',
|
||||
transferKind: 'push' as const,
|
||||
};
|
||||
(createDispatcher as jest.Mock).mockReturnValue(dispatcher);
|
||||
|
||||
const diagnostics = { report: jest.fn() };
|
||||
const provider = createRemoteStrapiDestinationProvider({
|
||||
strategy: 'restore',
|
||||
restore: {},
|
||||
url: new URL('http://localhost:1337/admin'),
|
||||
verifyChecksums: true,
|
||||
});
|
||||
|
||||
await provider.bootstrap(diagnostics as any);
|
||||
|
||||
expect(diagnostics.report).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
kind: 'warning',
|
||||
details: expect.objectContaining({
|
||||
message: expect.stringContaining('does not support checksum negotiation'),
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
const writable = provider.createAssetsWriteStream();
|
||||
if (writable instanceof Promise) {
|
||||
throw new Error('Expected synchronous Writable');
|
||||
}
|
||||
|
||||
const asset: IAsset = {
|
||||
filename: 'f.bin',
|
||||
filepath: '/tmp/f.bin',
|
||||
stats: { size: 3 } as IAsset['stats'],
|
||||
metadata: { id: 1 },
|
||||
stream: Readable.from([Buffer.from([1, 2, 3])]),
|
||||
};
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writable.write(asset, (err) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writable.end((err) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
|
||||
const endItem = streamBatches.flat().find((i) => (i as { action: string }).action === 'end') as
|
||||
| { checksum?: unknown }
|
||||
| undefined;
|
||||
|
||||
expect(endItem?.checksum).toBeUndefined();
|
||||
});
|
||||
});
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
import { createHash } from 'crypto';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
import type { IAsset } from '../../../../../types';
|
||||
|
||||
import { createRemoteStrapiDestinationProvider } from '..';
|
||||
import type { IRemoteStrapiDestinationProviderOptions } from '..';
|
||||
|
||||
const defaultOptions: IRemoteStrapiDestinationProviderOptions = {
|
||||
strategy: 'restore',
|
||||
url: new URL('http://localhost:1337/admin'),
|
||||
auth: undefined,
|
||||
};
|
||||
|
||||
function mockPushDispatcher() {
|
||||
const streamBatches: unknown[][] = [];
|
||||
return {
|
||||
streamBatches,
|
||||
dispatcher: {
|
||||
dispatchTransferStep: jest.fn(
|
||||
async (msg: { action: string; step?: string; data?: unknown }) => {
|
||||
if (msg.action === 'stream' && msg.step === 'assets' && Array.isArray(msg.data)) {
|
||||
streamBatches.push(msg.data);
|
||||
}
|
||||
if (msg.action === 'start') {
|
||||
return { ok: true };
|
||||
}
|
||||
if (msg.action === 'end') {
|
||||
return { ok: true, stats: { started: 1, finished: 1 } };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
),
|
||||
dispatchCommand: jest.fn(),
|
||||
dispatchTransferAction: jest.fn(),
|
||||
setTransferProperties: jest.fn(),
|
||||
dispatch: jest.fn(),
|
||||
transferID: 't1',
|
||||
transferKind: 'push' as const,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const writeOneAsset = async (writable: NodeJS.WritableStream, asset: IAsset): Promise<void> => {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writable.write(asset, (err: Error | null | undefined) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writable.end((err: Error | null | undefined) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
};
|
||||
|
||||
describe('Remote Strapi destination provider — push assets write stream', () => {
|
||||
test('flushes before asset completes when decoded payload exceeds 1MiB and uses base64 stream chunks', async () => {
|
||||
const { streamBatches, dispatcher } = mockPushDispatcher();
|
||||
const provider = createRemoteStrapiDestinationProvider(defaultOptions);
|
||||
provider.dispatcher = dispatcher as unknown as typeof provider.dispatcher;
|
||||
|
||||
const writable = provider.createAssetsWriteStream();
|
||||
if (writable instanceof Promise) {
|
||||
throw new Error('Expected synchronous Writable');
|
||||
}
|
||||
|
||||
const chunkSize = 200_000;
|
||||
const numChunks = 8;
|
||||
const stream = Readable.from(
|
||||
Array.from({ length: numChunks }, () => Buffer.alloc(chunkSize, 0xab))
|
||||
);
|
||||
|
||||
const asset: IAsset = {
|
||||
filename: 'big.bin',
|
||||
filepath: '/tmp/big.bin',
|
||||
stats: { size: chunkSize * numChunks } as IAsset['stats'],
|
||||
metadata: { id: 1 },
|
||||
stream,
|
||||
};
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writable.write(asset, (err) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writable.end((err) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
|
||||
expect(streamBatches.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const streamItems = streamBatches
|
||||
.flat()
|
||||
.filter((i) => (i as { action: string }).action === 'stream');
|
||||
expect(streamItems).toHaveLength(numChunks);
|
||||
for (const item of streamItems) {
|
||||
const row = item as { action: string; encoding?: string; data: unknown };
|
||||
expect(row.encoding).toBe('base64');
|
||||
expect(typeof row.data).toBe('string');
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
test('pushes many small assets with one start/end per asset (batches flushed, not accumulated)', async () => {
|
||||
const { streamBatches, dispatcher } = mockPushDispatcher();
|
||||
const provider = createRemoteStrapiDestinationProvider(defaultOptions);
|
||||
provider.dispatcher = dispatcher as unknown as typeof provider.dispatcher;
|
||||
|
||||
const writable = provider.createAssetsWriteStream();
|
||||
if (writable instanceof Promise) {
|
||||
throw new Error('Expected synchronous Writable');
|
||||
}
|
||||
|
||||
const assetCount = 30;
|
||||
|
||||
for (let i = 0; i < assetCount; i += 1) {
|
||||
const stream = Readable.from([Buffer.from([i % 256, 0x02, 0x03])]);
|
||||
const asset: IAsset = {
|
||||
filename: `f-${i}.bin`,
|
||||
filepath: `/tmp/f-${i}.bin`,
|
||||
stats: { size: 3 } as IAsset['stats'],
|
||||
metadata: { id: i },
|
||||
stream,
|
||||
};
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writable.write(asset, (err) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writable.end((err) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
|
||||
const flat = streamBatches.flat() as { action: string }[];
|
||||
expect(flat.filter((i) => i.action === 'start')).toHaveLength(assetCount);
|
||||
expect(flat.filter((i) => i.action === 'end')).toHaveLength(assetCount);
|
||||
expect(streamBatches.length).toBeGreaterThan(0);
|
||||
expect(dispatcher.dispatchTransferStep).toHaveBeenCalled();
|
||||
}, 30_000);
|
||||
|
||||
test('includes per-asset checksum in end row when verifyChecksums is enabled', async () => {
|
||||
const { streamBatches, dispatcher } = mockPushDispatcher();
|
||||
const provider = createRemoteStrapiDestinationProvider({
|
||||
...defaultOptions,
|
||||
verifyChecksums: true,
|
||||
});
|
||||
provider.dispatcher = dispatcher as unknown as typeof provider.dispatcher;
|
||||
|
||||
const writable = provider.createAssetsWriteStream();
|
||||
if (writable instanceof Promise) {
|
||||
throw new Error('Expected synchronous Writable');
|
||||
}
|
||||
|
||||
const payload = [Buffer.from('hello'), Buffer.from(' world')];
|
||||
const stream = Readable.from(payload);
|
||||
const asset: IAsset = {
|
||||
filename: 'checksum.bin',
|
||||
filepath: '/tmp/checksum.bin',
|
||||
stats: { size: 11 } as IAsset['stats'],
|
||||
metadata: { id: 1 },
|
||||
stream,
|
||||
};
|
||||
|
||||
await writeOneAsset(writable, asset);
|
||||
|
||||
const endItem = streamBatches.flat().find((i) => (i as { action: string }).action === 'end') as
|
||||
| { checksum?: { algorithm: string; value: string } }
|
||||
| undefined;
|
||||
|
||||
const expected = createHash('sha256').update(Buffer.concat(payload)).digest('hex');
|
||||
expect(endItem?.checksum).toEqual({ algorithm: 'sha256', value: expected });
|
||||
});
|
||||
|
||||
test('does not include checksum in end row when verifyChecksums is disabled', async () => {
|
||||
const { streamBatches, dispatcher } = mockPushDispatcher();
|
||||
const provider = createRemoteStrapiDestinationProvider({
|
||||
...defaultOptions,
|
||||
verifyChecksums: false,
|
||||
});
|
||||
provider.dispatcher = dispatcher as unknown as typeof provider.dispatcher;
|
||||
|
||||
const writable = provider.createAssetsWriteStream();
|
||||
if (writable instanceof Promise) {
|
||||
throw new Error('Expected synchronous Writable');
|
||||
}
|
||||
|
||||
const stream = Readable.from([Buffer.from('abc')]);
|
||||
const asset: IAsset = {
|
||||
filename: 'no-checksum.bin',
|
||||
filepath: '/tmp/no-checksum.bin',
|
||||
stats: { size: 3 } as IAsset['stats'],
|
||||
metadata: { id: 1 },
|
||||
stream,
|
||||
};
|
||||
|
||||
await writeOneAsset(writable, asset);
|
||||
|
||||
const endItem = streamBatches.flat().find((i) => (i as { action: string }).action === 'end') as
|
||||
| { checksum?: { algorithm: string; value: string } }
|
||||
| undefined;
|
||||
|
||||
expect(endItem?.checksum).toBeUndefined();
|
||||
});
|
||||
});
|
||||
+36
-6
@@ -3,13 +3,22 @@ import { TRANSFER_PATH } from '../../../remote/constants';
|
||||
import { CommandMessage } from '../../../../../types/remote/protocol/client';
|
||||
import { createDispatcher } from '../../utils';
|
||||
|
||||
jest.useFakeTimers();
|
||||
jest.mock('ws', () => ({
|
||||
WebSocket: jest.fn().mockImplementation(() => {
|
||||
let onMessage: ((data: string) => void) | undefined;
|
||||
return {
|
||||
...jest.requireActual('ws').WebSocket,
|
||||
send: jest.fn(),
|
||||
once: jest.fn(),
|
||||
send: jest.fn((payload: string, cb?: (err?: Error) => void) => {
|
||||
cb?.();
|
||||
const parsed = JSON.parse(payload) as { uuid: string };
|
||||
queueMicrotask(() => {
|
||||
onMessage?.(JSON.stringify({ uuid: parsed.uuid, data: null }));
|
||||
});
|
||||
}),
|
||||
once: jest.fn((event: string, handler: (data: string) => void) => {
|
||||
if (event === 'message') {
|
||||
onMessage = handler;
|
||||
}
|
||||
}),
|
||||
};
|
||||
}),
|
||||
}));
|
||||
@@ -19,14 +28,14 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe('Remote Strapi Destination Utils', () => {
|
||||
test('Dispatch method sends payload', () => {
|
||||
test('Dispatch method sends payload', async () => {
|
||||
const ws = new WebSocket(`ws://test/admin${TRANSFER_PATH}`);
|
||||
const message: CommandMessage = {
|
||||
type: 'command',
|
||||
command: 'status',
|
||||
};
|
||||
|
||||
createDispatcher(ws).dispatch(message);
|
||||
await createDispatcher(ws).dispatch(message);
|
||||
|
||||
expect.extend({
|
||||
toContain(receivedString, expected) {
|
||||
@@ -44,4 +53,25 @@ describe('Remote Strapi Destination Utils', () => {
|
||||
// @ts-ignore
|
||||
expect(ws.send).toHaveBeenCalledWith(expect.toContain(message), expect.anything());
|
||||
});
|
||||
|
||||
test('dispatch stringifies typed arrays with websocket replacer', async () => {
|
||||
const ws = new WebSocket(`ws://test/admin${TRANSFER_PATH}`);
|
||||
const message = {
|
||||
type: 'transfer' as const,
|
||||
kind: 'step' as const,
|
||||
step: 'entities' as const,
|
||||
action: 'stream' as const,
|
||||
data: [{ bytes: new Uint8Array([0xde, 0xad]) }],
|
||||
};
|
||||
|
||||
await createDispatcher(ws).dispatch(message);
|
||||
|
||||
// @ts-ignore
|
||||
const payload = (ws.send as jest.Mock).mock.calls[0]?.[0];
|
||||
const parsed = JSON.parse(payload) as {
|
||||
data: Array<{ bytes: string }>;
|
||||
};
|
||||
|
||||
expect(parsed.data[0].bytes).toBe(Buffer.from([0xde, 0xad]).toString('base64'));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import { Writable } from 'stream';
|
||||
import { WebSocket } from 'ws';
|
||||
import { once } from 'lodash/fp';
|
||||
@@ -19,6 +19,10 @@ import type { Client, Server, Auth } from '../../../../types/remote/protocol';
|
||||
import type { ILocalStrapiDestinationProviderOptions } from '../local-destination';
|
||||
import { TRANSFER_PATH } from '../../remote/constants';
|
||||
import { ProviderTransferError, ProviderValidationError } from '../../../errors/providers';
|
||||
import {
|
||||
createTransferAssetStreamChunk,
|
||||
transferAssetStreamChunkByteLength,
|
||||
} from '../../../utils/transfer-asset-chunk';
|
||||
|
||||
export interface IRemoteStrapiDestinationProviderOptions
|
||||
extends Pick<ILocalStrapiDestinationProviderOptions, 'restore' | 'strategy'> {
|
||||
@@ -28,6 +32,8 @@ export interface IRemoteStrapiDestinationProviderOptions
|
||||
retryMessageTimeout: number; // milliseconds to wait for a response from a message
|
||||
retryMessageMaxRetries: number; // max number of retries for a message before aborting transfer
|
||||
};
|
||||
/** Include per-asset stream checksums and require peers to validate on receive. */
|
||||
verifyChecksums?: boolean;
|
||||
}
|
||||
|
||||
const jsonLength = (obj: object) => Buffer.byteLength(JSON.stringify(obj));
|
||||
@@ -49,11 +55,14 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
|
||||
|
||||
#diagnostics?: IDiagnosticReporter;
|
||||
|
||||
#checksumsEnabled = false;
|
||||
|
||||
constructor(options: IRemoteStrapiDestinationProviderOptions) {
|
||||
this.options = options;
|
||||
this.ws = null;
|
||||
this.dispatcher = null;
|
||||
this.transferID = null;
|
||||
this.#checksumsEnabled = options.verifyChecksums === true;
|
||||
|
||||
this.resetStats();
|
||||
}
|
||||
@@ -69,16 +78,29 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
|
||||
|
||||
async initTransfer(): Promise<string> {
|
||||
const { strategy, restore } = this.options;
|
||||
const wantsChecksums = this.options.verifyChecksums === true;
|
||||
|
||||
const query = this.dispatcher?.dispatchCommand({
|
||||
command: 'init',
|
||||
params: { options: { strategy, restore }, transfer: 'push' },
|
||||
params: {
|
||||
options: { strategy, restore },
|
||||
transfer: 'push',
|
||||
...(wantsChecksums ? { checksums: true } : {}),
|
||||
},
|
||||
});
|
||||
|
||||
const res = (await query) as Server.Payload<Server.InitMessage>;
|
||||
const res = (await query) as
|
||||
| (Server.Payload<Server.InitMessage> & { checksums?: boolean })
|
||||
| null;
|
||||
if (!res?.transferID) {
|
||||
throw new ProviderTransferError('Init failed, invalid response from the server');
|
||||
}
|
||||
this.#checksumsEnabled = wantsChecksums && res.checksums === true;
|
||||
if (wantsChecksums && res.checksums !== true) {
|
||||
this.#reportWarning(
|
||||
'[Data transfer][push] Checksums were requested but the remote does not support checksum negotiation; continuing without checksum validation.'
|
||||
);
|
||||
}
|
||||
|
||||
this.resetStats();
|
||||
|
||||
@@ -233,6 +255,17 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
|
||||
});
|
||||
}
|
||||
|
||||
#reportWarning(message: string) {
|
||||
this.#diagnostics?.report({
|
||||
details: {
|
||||
createdAt: new Date(),
|
||||
message,
|
||||
origin: 'remote-destination-provider',
|
||||
},
|
||||
kind: 'warning',
|
||||
});
|
||||
}
|
||||
|
||||
async bootstrap(diagnostics?: IDiagnosticReporter): Promise<void> {
|
||||
this.#diagnostics = diagnostics;
|
||||
const { url, auth } = this.options;
|
||||
@@ -354,13 +387,11 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
|
||||
createAssetsWriteStream(): Writable | Promise<Writable> {
|
||||
let batch: Client.TransferAssetFlow[] = [];
|
||||
let hasStarted = false;
|
||||
const verifyChecksums = this.#checksumsEnabled;
|
||||
|
||||
const batchSize = 1024 * 1024; // 1MB;
|
||||
const batchLength = () => {
|
||||
return batch.reduce(
|
||||
(acc, chunk) => (chunk.action === 'stream' ? acc + chunk.data.byteLength : acc),
|
||||
0
|
||||
);
|
||||
return batch.reduce((acc, chunk) => acc + transferAssetStreamChunkByteLength(chunk), 0);
|
||||
};
|
||||
const startAssetsTransferOnce = this.#startStepOnce('assets');
|
||||
|
||||
@@ -409,6 +440,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
|
||||
|
||||
const assetID = randomUUID();
|
||||
const { filename, filepath, stats, stream, metadata } = asset;
|
||||
const checksumHash = verifyChecksums ? createHash('sha256') : undefined;
|
||||
|
||||
try {
|
||||
await safePush({
|
||||
@@ -418,16 +450,21 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider {
|
||||
});
|
||||
|
||||
for await (const chunk of stream) {
|
||||
await safePush({ action: 'stream', assetID, data: chunk });
|
||||
checksumHash?.update(chunk);
|
||||
await safePush(createTransferAssetStreamChunk(assetID, chunk));
|
||||
}
|
||||
|
||||
await safePush({ action: 'end', assetID });
|
||||
await safePush({
|
||||
action: 'end',
|
||||
assetID,
|
||||
...(checksumHash
|
||||
? { checksum: { algorithm: 'sha256' as const, value: checksumHash.digest('hex') } }
|
||||
: {}),
|
||||
});
|
||||
|
||||
callback();
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
callback(error);
|
||||
}
|
||||
callback(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { Writable } from 'stream';
|
||||
import type { Core } from '@strapi/types';
|
||||
|
||||
import type { IAsset } from '../../../../../types';
|
||||
import { createRemoteStrapiSourceProvider } from '..';
|
||||
|
||||
import { connectToWebsocket, createDispatcher } from '../../utils';
|
||||
|
||||
jest.mock('../../utils', () => ({
|
||||
...jest.requireActual('../../utils'),
|
||||
connectToWebsocket: jest.fn(),
|
||||
createDispatcher: jest.fn(),
|
||||
}));
|
||||
|
||||
class MockWebSocket extends EventEmitter {
|
||||
send = jest.fn((_payload: string, cb?: (err?: Error) => void) => cb?.());
|
||||
|
||||
close = jest.fn();
|
||||
}
|
||||
|
||||
const transferMessageBuffer = (opts: {
|
||||
uuid: string;
|
||||
processId: string;
|
||||
ended?: boolean;
|
||||
error?: unknown;
|
||||
data?: unknown;
|
||||
}) =>
|
||||
Buffer.from(
|
||||
JSON.stringify({
|
||||
uuid: opts.uuid,
|
||||
data: {
|
||||
type: 'transfer',
|
||||
id: opts.processId,
|
||||
ended: opts.ended ?? false,
|
||||
error: opts.error ?? null,
|
||||
data: opts.data ?? null,
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
const oneAssetBatch = (assetID: string) => [
|
||||
{
|
||||
action: 'start' as const,
|
||||
assetID,
|
||||
data: {
|
||||
filename: 't.bin',
|
||||
metadata: { id: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
action: 'stream' as const,
|
||||
assetID,
|
||||
data: { type: 'Buffer' as const, data: [1, 2, 3] },
|
||||
},
|
||||
{ action: 'end' as const, assetID },
|
||||
];
|
||||
|
||||
describe('Remote source checksum negotiation', () => {
|
||||
test('warns and continues without checksums when peer does not support checksum negotiation', async () => {
|
||||
const ws = new MockWebSocket();
|
||||
(connectToWebsocket as jest.Mock).mockResolvedValue(ws);
|
||||
|
||||
const dispatcher = {
|
||||
dispatchCommand: jest.fn().mockResolvedValue({ transferID: 't1' }), // no `checksums` support
|
||||
dispatchTransferAction: jest.fn().mockResolvedValue(null),
|
||||
dispatchTransferStep: jest.fn(async (msg: { action: string }) => {
|
||||
if (msg.action === 'start') {
|
||||
return { id: 'pid-1' };
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
setTransferProperties: jest.fn(),
|
||||
dispatch: jest.fn(),
|
||||
transferID: 't1',
|
||||
transferKind: 'pull' as const,
|
||||
};
|
||||
(createDispatcher as jest.Mock).mockReturnValue(dispatcher);
|
||||
|
||||
const diagnostics = { report: jest.fn() };
|
||||
const provider = createRemoteStrapiSourceProvider({
|
||||
url: new URL('http://localhost:1337/admin'),
|
||||
getStrapi: () => ({}) as Core.Strapi,
|
||||
verifyChecksums: true,
|
||||
});
|
||||
|
||||
await provider.bootstrap(diagnostics as any);
|
||||
|
||||
expect(diagnostics.report).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
kind: 'warning',
|
||||
details: expect.objectContaining({
|
||||
message: expect.stringContaining('does not support checksum negotiation'),
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
const pass = await provider.createAssetsReadStream();
|
||||
const drainPass = pipeline(
|
||||
pass,
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
write(asset: IAsset, _enc, cb) {
|
||||
pipeline(
|
||||
asset.stream,
|
||||
new Writable({
|
||||
write(_chunk, _e, c) {
|
||||
c();
|
||||
},
|
||||
})
|
||||
).then(
|
||||
() => cb(),
|
||||
(err) => cb(err)
|
||||
);
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: 'u-1',
|
||||
processId: 'pid-1',
|
||||
data: [oneAssetBatch('a1')],
|
||||
})
|
||||
);
|
||||
ws.emit('message', transferMessageBuffer({ uuid: 'u-end', processId: 'pid-1', ended: true }));
|
||||
|
||||
await expect(drainPass).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
+506
@@ -0,0 +1,506 @@
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { createHash } from 'crypto';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { Writable } from 'stream';
|
||||
import type { Core } from '@strapi/types';
|
||||
import type { WebSocket } from 'ws';
|
||||
|
||||
import type { IAsset } from '../../../../../types';
|
||||
|
||||
import { createRemoteStrapiSourceProvider } from '..';
|
||||
import { ProviderTransferError } from '../../../../errors/providers';
|
||||
|
||||
/** WebSocket frame shape expected by RemoteStrapiSourceProvider.#createStageReadStream */
|
||||
function transferMessageBuffer(opts: {
|
||||
uuid: string;
|
||||
processId: string;
|
||||
ended?: boolean;
|
||||
error?: unknown;
|
||||
data?: unknown;
|
||||
}) {
|
||||
return Buffer.from(
|
||||
JSON.stringify({
|
||||
uuid: opts.uuid,
|
||||
data: {
|
||||
type: 'transfer',
|
||||
id: opts.processId,
|
||||
ended: opts.ended ?? false,
|
||||
error: opts.error ?? null,
|
||||
data: opts.data ?? null,
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
function oneAssetBatch(
|
||||
assetID: string,
|
||||
checksum?: {
|
||||
algorithm: 'sha256';
|
||||
value: string;
|
||||
}
|
||||
) {
|
||||
const chunk = Buffer.from([0x01, 0x02, 0x03]);
|
||||
return [
|
||||
{
|
||||
action: 'start' as const,
|
||||
assetID,
|
||||
data: {
|
||||
filename: 't.bin',
|
||||
metadata: { id: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
action: 'stream' as const,
|
||||
assetID,
|
||||
data: { type: 'Buffer' as const, data: Array.from(chunk) },
|
||||
},
|
||||
{ action: 'end' as const, assetID, ...(checksum ? { checksum } : {}) },
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal WebSocket mock: provider only needs `once('message')` and `send` for ACKs.
|
||||
*/
|
||||
class MockTransferWebSocket extends EventEmitter {
|
||||
send = jest.fn((_payload: string, cb?: (err?: Error) => void) => {
|
||||
cb?.();
|
||||
});
|
||||
}
|
||||
|
||||
const createProviderWithMockDispatcher = (
|
||||
ws: MockTransferWebSocket,
|
||||
processId: string,
|
||||
options: { verifyChecksums?: boolean } = {}
|
||||
) => {
|
||||
const provider = createRemoteStrapiSourceProvider({
|
||||
url: new URL('http://localhost:1337/admin'),
|
||||
getStrapi: () => ({}) as Core.Strapi,
|
||||
streamTimeout: 60_000,
|
||||
...(options.verifyChecksums !== undefined ? { verifyChecksums: options.verifyChecksums } : {}),
|
||||
});
|
||||
|
||||
provider.ws = ws as unknown as WebSocket;
|
||||
provider.dispatcher = {
|
||||
dispatchTransferStep: jest.fn(async (msg: { action: string }) => {
|
||||
if (msg.action === 'start') {
|
||||
return { id: processId };
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
setTransferProperties: jest.fn(),
|
||||
dispatchCommand: jest.fn(),
|
||||
dispatch: jest.fn(),
|
||||
dispatchTransferAction: jest.fn(),
|
||||
transferID: 't1',
|
||||
transferKind: 'pull',
|
||||
} as unknown as typeof provider.dispatcher;
|
||||
|
||||
return provider;
|
||||
};
|
||||
|
||||
describe('Remote Strapi source provider — pull assets stream', () => {
|
||||
const processId = 'flush-process-id';
|
||||
|
||||
const flushPromises = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
setImmediate(() => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
test('after each asset completes the same assetID can start again (registry must not retain closed assets)', async () => {
|
||||
const ws = new MockTransferWebSocket();
|
||||
const dispatchTransferStep = jest.fn(async (msg: { action: string; step?: string }) => {
|
||||
if (msg.action === 'start') {
|
||||
return { id: processId };
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const provider = createRemoteStrapiSourceProvider({
|
||||
url: new URL('http://localhost:1337/admin'),
|
||||
getStrapi: () => ({}) as Core.Strapi,
|
||||
streamTimeout: 60_000,
|
||||
});
|
||||
|
||||
provider.ws = ws as unknown as WebSocket;
|
||||
provider.dispatcher = {
|
||||
dispatchTransferStep,
|
||||
setTransferProperties: jest.fn(),
|
||||
dispatchCommand: jest.fn(),
|
||||
dispatch: jest.fn(),
|
||||
dispatchTransferAction: jest.fn(),
|
||||
transferID: 't1',
|
||||
transferKind: 'pull',
|
||||
} as unknown as typeof provider.dispatcher;
|
||||
|
||||
const pass = await provider.createAssetsReadStream();
|
||||
|
||||
const reuseId = 'reused-asset-id';
|
||||
const cycles = 40;
|
||||
|
||||
const drainPass = pipeline(
|
||||
pass,
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(asset: IAsset, _enc, cb) {
|
||||
const rs = asset.stream;
|
||||
if (!rs) {
|
||||
cb();
|
||||
return;
|
||||
}
|
||||
pipeline(
|
||||
rs,
|
||||
new Writable({
|
||||
write(_chunk, _e, c) {
|
||||
c();
|
||||
},
|
||||
})
|
||||
).then(
|
||||
() => {
|
||||
cb();
|
||||
},
|
||||
(err) => {
|
||||
cb(err);
|
||||
}
|
||||
);
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
for (let i = 0; i < cycles; i += 1) {
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: `u-${i}`,
|
||||
processId,
|
||||
data: [oneAssetBatch(reuseId)],
|
||||
})
|
||||
);
|
||||
await flushPromises();
|
||||
}
|
||||
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: 'u-end',
|
||||
processId,
|
||||
ended: true,
|
||||
})
|
||||
);
|
||||
|
||||
await drainPass;
|
||||
|
||||
// Client start + end for the assets step, and one WebSocket ACK per incoming message (cycles + end).
|
||||
expect(dispatchTransferStep).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ action: 'start', step: 'assets' }),
|
||||
expect.objectContaining({
|
||||
retryOverrides: expect.objectContaining({
|
||||
retryMessageTimeout: 120_000,
|
||||
retryMessageMaxRetries: 30,
|
||||
}),
|
||||
})
|
||||
);
|
||||
expect(dispatchTransferStep).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ action: 'end', step: 'assets' })
|
||||
);
|
||||
expect(dispatchTransferStep).toHaveBeenCalledTimes(2);
|
||||
expect(ws.send).toHaveBeenCalledTimes(cycles + 1);
|
||||
}, 15_000);
|
||||
|
||||
test('many small asset batches complete without error (registry does not grow with batch count)', async () => {
|
||||
const ws = new MockTransferWebSocket();
|
||||
const dispatchTransferStep = jest.fn(async (msg: { action: string }) => {
|
||||
if (msg.action === 'start') {
|
||||
return { id: processId };
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const provider = createRemoteStrapiSourceProvider({
|
||||
url: new URL('http://localhost:1337/admin'),
|
||||
getStrapi: () => ({}) as Core.Strapi,
|
||||
streamTimeout: 60_000,
|
||||
});
|
||||
|
||||
provider.ws = ws as unknown as WebSocket;
|
||||
provider.dispatcher = {
|
||||
dispatchTransferStep,
|
||||
setTransferProperties: jest.fn(),
|
||||
dispatchCommand: jest.fn(),
|
||||
dispatch: jest.fn(),
|
||||
dispatchTransferAction: jest.fn(),
|
||||
transferID: 't1',
|
||||
transferKind: 'pull',
|
||||
} as unknown as typeof provider.dispatcher;
|
||||
|
||||
const pass = await provider.createAssetsReadStream();
|
||||
|
||||
const batchCount = 25;
|
||||
|
||||
const drainPass = pipeline(
|
||||
pass,
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(asset: IAsset, _enc, cb) {
|
||||
const rs = asset.stream;
|
||||
if (!rs) {
|
||||
cb();
|
||||
return;
|
||||
}
|
||||
pipeline(
|
||||
rs,
|
||||
new Writable({
|
||||
write(_chunk, _e, c) {
|
||||
c();
|
||||
},
|
||||
})
|
||||
).then(
|
||||
() => {
|
||||
cb();
|
||||
},
|
||||
(err) => {
|
||||
cb(err);
|
||||
}
|
||||
);
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
for (let i = 0; i < batchCount; i += 1) {
|
||||
const id = `unique-${i}`;
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: `batch-${i}`,
|
||||
processId,
|
||||
data: [oneAssetBatch(id)],
|
||||
})
|
||||
);
|
||||
await flushPromises();
|
||||
}
|
||||
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: 'u-end',
|
||||
processId,
|
||||
ended: true,
|
||||
})
|
||||
);
|
||||
|
||||
await drainPass;
|
||||
|
||||
expect(dispatchTransferStep).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ action: 'start', step: 'assets' }),
|
||||
expect.objectContaining({
|
||||
retryOverrides: expect.objectContaining({
|
||||
retryMessageTimeout: 120_000,
|
||||
retryMessageMaxRetries: 30,
|
||||
}),
|
||||
})
|
||||
);
|
||||
expect(dispatchTransferStep).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ action: 'end', step: 'assets' })
|
||||
);
|
||||
expect(dispatchTransferStep).toHaveBeenCalledTimes(2);
|
||||
expect(ws.send).toHaveBeenCalledTimes(batchCount + 1);
|
||||
}, 20_000);
|
||||
|
||||
test('throws on checksum mismatch when verifyChecksums is enabled', async () => {
|
||||
const ws = new MockTransferWebSocket();
|
||||
const provider = createProviderWithMockDispatcher(ws, processId, { verifyChecksums: true });
|
||||
|
||||
const pass = await provider.createAssetsReadStream();
|
||||
const id = 'asset-with-bad-checksum';
|
||||
const badChecksum = createHash('sha256').update(Buffer.from('wrong')).digest('hex');
|
||||
|
||||
const drainPass = pipeline(
|
||||
pass,
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
write(asset: IAsset, _enc, cb) {
|
||||
pipeline(
|
||||
asset.stream,
|
||||
new Writable({
|
||||
write(_chunk, _e, c) {
|
||||
c();
|
||||
},
|
||||
})
|
||||
).then(
|
||||
() => cb(),
|
||||
(err) => cb(err)
|
||||
);
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: 'checksum-1',
|
||||
processId,
|
||||
data: [
|
||||
oneAssetBatch(id, {
|
||||
algorithm: 'sha256',
|
||||
value: badChecksum,
|
||||
}),
|
||||
],
|
||||
})
|
||||
);
|
||||
await flushPromises();
|
||||
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: 'checksum-end',
|
||||
processId,
|
||||
ended: true,
|
||||
})
|
||||
);
|
||||
|
||||
const checksumErr = await drainPass.catch((e: unknown) => e);
|
||||
expect(checksumErr).toBeInstanceOf(ProviderTransferError);
|
||||
expect((checksumErr as Error).message).toMatch(/Checksum mismatch/);
|
||||
});
|
||||
|
||||
test('throws ProviderTransferError when checksum is required but missing', async () => {
|
||||
const ws = new MockTransferWebSocket();
|
||||
const provider = createProviderWithMockDispatcher(ws, processId, { verifyChecksums: true });
|
||||
|
||||
const pass = await provider.createAssetsReadStream();
|
||||
const id = 'asset-missing-checksum';
|
||||
|
||||
const drainPass = pipeline(
|
||||
pass,
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
write(asset: IAsset, _enc, cb) {
|
||||
pipeline(
|
||||
asset.stream,
|
||||
new Writable({
|
||||
write(_chunk, _e, c) {
|
||||
c();
|
||||
},
|
||||
})
|
||||
).then(
|
||||
() => cb(),
|
||||
(err) => cb(err)
|
||||
);
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: 'missing-1',
|
||||
processId,
|
||||
data: [oneAssetBatch(id)],
|
||||
})
|
||||
);
|
||||
await flushPromises();
|
||||
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: 'missing-end',
|
||||
processId,
|
||||
ended: true,
|
||||
})
|
||||
);
|
||||
|
||||
const missingErr = await drainPass.catch((e: unknown) => e);
|
||||
expect(missingErr).toBeInstanceOf(ProviderTransferError);
|
||||
expect((missingErr as Error).message).toMatch(/missing checksum/i);
|
||||
});
|
||||
|
||||
test('accepts matching checksum when verifyChecksums is enabled', async () => {
|
||||
const ws = new MockTransferWebSocket();
|
||||
const provider = createProviderWithMockDispatcher(ws, processId, { verifyChecksums: true });
|
||||
const pass = await provider.createAssetsReadStream();
|
||||
|
||||
const id = 'asset-with-good-checksum';
|
||||
const goodChecksum = createHash('sha256')
|
||||
.update(Buffer.from([0x01, 0x02, 0x03]))
|
||||
.digest('hex');
|
||||
|
||||
const drainPass = pipeline(
|
||||
pass,
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
write(asset: IAsset, _enc, cb) {
|
||||
pipeline(
|
||||
asset.stream,
|
||||
new Writable({
|
||||
write(_chunk, _e, c) {
|
||||
c();
|
||||
},
|
||||
})
|
||||
).then(
|
||||
() => cb(),
|
||||
(err) => cb(err)
|
||||
);
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: 'good-1',
|
||||
processId,
|
||||
data: [
|
||||
oneAssetBatch(id, {
|
||||
algorithm: 'sha256',
|
||||
value: goodChecksum,
|
||||
}),
|
||||
],
|
||||
})
|
||||
);
|
||||
await flushPromises();
|
||||
ws.emit('message', transferMessageBuffer({ uuid: 'good-end', processId, ended: true }));
|
||||
|
||||
await expect(drainPass).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test('ignores missing checksum when verifyChecksums is disabled', async () => {
|
||||
const ws = new MockTransferWebSocket();
|
||||
const provider = createProviderWithMockDispatcher(ws, processId, { verifyChecksums: false });
|
||||
const pass = await provider.createAssetsReadStream();
|
||||
|
||||
const id = 'asset-without-checksum-disabled';
|
||||
const drainPass = pipeline(
|
||||
pass,
|
||||
new Writable({
|
||||
objectMode: true,
|
||||
write(asset: IAsset, _enc, cb) {
|
||||
pipeline(
|
||||
asset.stream,
|
||||
new Writable({
|
||||
write(_chunk, _e, c) {
|
||||
c();
|
||||
},
|
||||
})
|
||||
).then(
|
||||
() => cb(),
|
||||
(err) => cb(err)
|
||||
);
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
ws.emit(
|
||||
'message',
|
||||
transferMessageBuffer({
|
||||
uuid: 'disabled-1',
|
||||
processId,
|
||||
data: [oneAssetBatch(id)],
|
||||
})
|
||||
);
|
||||
await flushPromises();
|
||||
ws.emit('message', transferMessageBuffer({ uuid: 'disabled-end', processId, ended: true }));
|
||||
|
||||
await expect(drainPass).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash, type Hash } from 'crypto';
|
||||
import { PassThrough, Readable, Writable } from 'stream';
|
||||
import type { Struct, Utils } from '@strapi/types';
|
||||
import { WebSocket } from 'ws';
|
||||
@@ -11,14 +12,32 @@ import type {
|
||||
MaybePromise,
|
||||
Protocol,
|
||||
ProviderType,
|
||||
StageTotalsEstimate,
|
||||
TransferStage,
|
||||
} from '../../../../types';
|
||||
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
|
||||
import { Client, Server, Auth } from '../../../../types/remote/protocol';
|
||||
import { ProviderTransferError, ProviderValidationError } from '../../../errors/providers';
|
||||
import { TRANSFER_PATH } from '../../remote/constants';
|
||||
import { decodeTransferAssetStreamItem } from '../../../utils/transfer-asset-chunk';
|
||||
import { write } from '../../../utils/writable-async-write';
|
||||
import { ILocalStrapiSourceProviderOptions } from '../local-source';
|
||||
import { createDispatcher, connectToWebsocket, trimTrailingSlash } from '../utils';
|
||||
import {
|
||||
createDispatcher,
|
||||
connectToWebsocket,
|
||||
trimTrailingSlash,
|
||||
type RetryMessageOptions,
|
||||
} from '../utils';
|
||||
|
||||
/**
|
||||
* Pull server answers `assets` step `start` only after `estimateAssetTotals` (DB stream; remote sizes from DB when complete, else HTTP like `createAssetsStream`).
|
||||
* That can exceed the default dispatcher wait (~30s between resends, a few minutes total). This message
|
||||
* uses a longer window so large libraries do not fail with `Request timed out` before totals are returned.
|
||||
*/
|
||||
const ASSETS_START_RETRY_OVERRIDES: Partial<RetryMessageOptions> = {
|
||||
retryMessageTimeout: 120_000,
|
||||
retryMessageMaxRetries: 30,
|
||||
};
|
||||
|
||||
export interface IRemoteStrapiSourceProviderOptions extends ILocalStrapiSourceProviderOptions {
|
||||
url: URL; // the url of the remote Strapi admin
|
||||
@@ -27,7 +46,10 @@ export interface IRemoteStrapiSourceProviderOptions extends ILocalStrapiSourcePr
|
||||
retryMessageTimeout: number; // milliseconds to wait for a response from a message
|
||||
retryMessageMaxRetries: number; // max number of retries for a message before aborting transfer
|
||||
};
|
||||
streamTimeout?: number; // milliseconds to wait between chunks of an asset before aborting the transfer
|
||||
/** Max ms without forward progress on an asset (new remote chunk accepted or chunk fully handed to the asset stream). */
|
||||
streamTimeout?: number;
|
||||
/** Require per-asset checksum verification for transferred asset bytes. */
|
||||
verifyChecksums?: boolean;
|
||||
}
|
||||
|
||||
type QueueableAction = Protocol.Client.TransferAssetFlow &
|
||||
@@ -45,7 +67,8 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
dispatcher: ReturnType<typeof createDispatcher> | null;
|
||||
|
||||
defaultOptions: Partial<IRemoteStrapiSourceProviderOptions> = {
|
||||
streamTimeout: 15000,
|
||||
// Large files + JSON/WS backpressure can go minutes between *messages* while bytes still drain locally
|
||||
streamTimeout: 300_000,
|
||||
};
|
||||
|
||||
constructor(options: IRemoteStrapiSourceProviderOptions) {
|
||||
@@ -53,6 +76,7 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
...this.defaultOptions,
|
||||
...options,
|
||||
};
|
||||
this.#checksumsEnabled = this.options.verifyChecksums === true;
|
||||
|
||||
this.ws = null;
|
||||
this.dispatcher = null;
|
||||
@@ -62,15 +86,36 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
|
||||
#diagnostics?: IDiagnosticReporter;
|
||||
|
||||
#pullAssetStreamWireSampleLogged = false;
|
||||
|
||||
#checksumsEnabled = false;
|
||||
|
||||
/** Set from pull server `start` response for `assets` when present (for engine `getStageTotals`). */
|
||||
#cachedAssetsTotals?: StageTotalsEstimate;
|
||||
|
||||
async #createStageReadStream(stage: Exclude<TransferStage, 'schemas'>) {
|
||||
if (stage === 'assets') {
|
||||
this.#cachedAssetsTotals = undefined;
|
||||
}
|
||||
|
||||
const startResult = await this.#startStep(stage);
|
||||
|
||||
if (startResult instanceof Error) {
|
||||
throw startResult;
|
||||
}
|
||||
|
||||
const { id: processID } = startResult as { id: string };
|
||||
const { id: processID, totals } = startResult as {
|
||||
id: string;
|
||||
totals?: StageTotalsEstimate;
|
||||
};
|
||||
|
||||
if (stage === 'assets' && totals && (totals.totalBytes != null || totals.totalCount != null)) {
|
||||
this.#cachedAssetsTotals = totals;
|
||||
}
|
||||
|
||||
// Default object-mode HWM (~16 chunks). Do not await `drain` on manual `push` while `pipe()`
|
||||
// is attached — drain/`readableLength` races reliably deadlock after a few 1MiB asset frames.
|
||||
// Backpressure for pull assets is enforced by the Writable below (`highWaterMark: 1`).
|
||||
const stream = new PassThrough({ objectMode: true });
|
||||
|
||||
const listener = async (raw: Buffer) => {
|
||||
@@ -98,9 +143,8 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
return;
|
||||
}
|
||||
|
||||
// if we get a single items instead of a batch
|
||||
for (const item of castArray(data)) {
|
||||
stream.push(item);
|
||||
stream.push(item as Parameters<PassThrough['push']>[0]);
|
||||
}
|
||||
|
||||
this.ws?.once('message', listener);
|
||||
@@ -121,18 +165,6 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
return this.#createStageReadStream('links');
|
||||
}
|
||||
|
||||
writeAsync = <T>(stream: Writable, data: T) => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
stream.write(data, (error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
}
|
||||
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
async createAssetsReadStream(): Promise<Readable> {
|
||||
// Create the streams used to transfer the assets
|
||||
const stream = await this.#createStageReadStream('assets');
|
||||
@@ -146,93 +178,150 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
queue: Array<QueueableAction>;
|
||||
status: 'ok' | 'closed' | 'errored';
|
||||
timeout?: NodeJS.Timeout;
|
||||
checksumHash?: Hash;
|
||||
};
|
||||
} = {};
|
||||
|
||||
// Watch for stalled assets; if we don't receive a chunk within timeout, abort transfer
|
||||
// Watch for stalled assets: no remote chunks and no completed writes to the asset stream for streamTimeout ms
|
||||
const resetTimeout = (assetID: string) => {
|
||||
if (!assets[assetID]) {
|
||||
return;
|
||||
}
|
||||
if (assets[assetID].timeout) {
|
||||
clearTimeout(assets[assetID].timeout);
|
||||
}
|
||||
assets[assetID].timeout = setTimeout(() => {
|
||||
if (!assets[assetID]) {
|
||||
return;
|
||||
}
|
||||
this.#reportInfo(`Asset ${assetID} transfer stalled, aborting.`);
|
||||
assets[assetID].status = 'errored';
|
||||
assets[assetID].stream.destroy(new Error(`Asset ${assetID} transfer timed out`));
|
||||
}, this.options.streamTimeout);
|
||||
};
|
||||
|
||||
stream
|
||||
/**
|
||||
* Process a payload of many transfer assets and performs the following tasks:
|
||||
* - Start: creates a stream for new assets.
|
||||
* - Stream: writes asset chunks to the asset's stream.
|
||||
* - End: closes the stream after the asset s transferred and cleanup related resources.
|
||||
*/
|
||||
.on('data', async (payload: Protocol.Client.TransferAssetFlow[]) => {
|
||||
for (const item of payload) {
|
||||
const { action, assetID } = item;
|
||||
const clearStallTimeoutForAsset = (assetID: string) => {
|
||||
const entry = assets[assetID];
|
||||
if (entry?.timeout) {
|
||||
clearTimeout(entry.timeout);
|
||||
entry.timeout = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
// Creates the stream to send the incoming asset through
|
||||
if (action === 'start') {
|
||||
// if a transfer has already been started for the same asset ID, something is wrong
|
||||
if (assets[assetID]) {
|
||||
throw new Error(`Asset ${assetID} already started`);
|
||||
const clearAllStallTimeouts = () => {
|
||||
for (const id of Object.keys(assets)) {
|
||||
clearStallTimeoutForAsset(id);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Serialize asset batch handling: `Readable.on('data', async …)` does not apply backpressure,
|
||||
* so we pipe through a Writable with highWaterMark 1 so only one batch is in flight.
|
||||
*/
|
||||
const processAssetPayload = async (payload: Protocol.Client.TransferAssetFlow[]) => {
|
||||
for (const item of payload) {
|
||||
const { action, assetID } = item;
|
||||
|
||||
if (action === 'start') {
|
||||
if (assets[assetID]) {
|
||||
throw new Error(`Asset ${assetID} already started`);
|
||||
}
|
||||
|
||||
this.#reportInfo(`Asset ${assetID} starting`);
|
||||
assets[assetID] = {
|
||||
...item.data,
|
||||
stream: new PassThrough(),
|
||||
status: 'ok',
|
||||
queue: [],
|
||||
...(this.#checksumsEnabled ? { checksumHash: createHash('sha256') } : {}),
|
||||
};
|
||||
|
||||
resetTimeout(assetID);
|
||||
|
||||
await write(pass, assets[assetID]);
|
||||
} else if (action === 'stream' || action === 'end') {
|
||||
if (!assets[assetID]) {
|
||||
throw new Error(`No id matching ${assetID} for stream action`);
|
||||
}
|
||||
|
||||
if (action === 'stream') {
|
||||
if (!this.#pullAssetStreamWireSampleLogged) {
|
||||
this.#pullAssetStreamWireSampleLogged = true;
|
||||
const { data } = item;
|
||||
// Same legacy shape `decodeTransferAssetStreamData` accepts after JSON.parse (proof, not frame-size guess).
|
||||
const legacyBufferJson =
|
||||
data &&
|
||||
typeof data === 'object' &&
|
||||
!Buffer.isBuffer(data) &&
|
||||
(data as { type?: string }).type === 'Buffer' &&
|
||||
(Array.isArray((data as { data?: unknown }).data) ||
|
||||
ArrayBuffer.isView((data as { data?: unknown }).data));
|
||||
if (legacyBufferJson) {
|
||||
this.#reportWarning(
|
||||
'[Data transfer][pull] Remote is using legacy Buffer JSON for asset chunks (each byte as a JSON number). That uses much more memory during JSON.parse than base64. Upgrade the remote Strapi to a version that sends base64 asset chunks, or out-of-memory errors may still happen on large files.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.#reportInfo(`Asset ${assetID} starting`);
|
||||
// Register the asset
|
||||
assets[assetID] = {
|
||||
...item.data,
|
||||
stream: new PassThrough(),
|
||||
status: 'ok',
|
||||
queue: [],
|
||||
};
|
||||
|
||||
resetTimeout(assetID);
|
||||
|
||||
// Connect the individual asset stream to the main asset stage stream
|
||||
// Note: nothing is transferred until data chunks are fed to the asset stream
|
||||
await this.writeAsync(pass, assets[assetID]);
|
||||
} else {
|
||||
clearTimeout(assets[assetID].timeout);
|
||||
}
|
||||
|
||||
// Writes the asset's data chunks to their corresponding stream
|
||||
// "end" is considered a chunk, but it's not a data chunk, it's a control message
|
||||
// That is done so that we don't complicate the already complicated async processing of the queue
|
||||
else if (action === 'stream' || action === 'end') {
|
||||
// If the asset hasn't been registered, or if it's been closed already, something is wrong
|
||||
if (!assets[assetID]) {
|
||||
throw new Error(`No id matching ${assetID} for stream action`);
|
||||
}
|
||||
if (assets[assetID].status === 'closed') {
|
||||
throw new Error(`Asset ${assetID} is closed`);
|
||||
}
|
||||
|
||||
// On every action, reset the timeout timer
|
||||
if (action === 'stream') {
|
||||
resetTimeout(assetID);
|
||||
} else {
|
||||
clearTimeout(assets[assetID].timeout);
|
||||
}
|
||||
assets[assetID].queue.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
if (assets[assetID].status === 'closed') {
|
||||
throw new Error(`Asset ${assetID} is closed`);
|
||||
}
|
||||
|
||||
assets[assetID].queue.push(item);
|
||||
for (const assetID in assets) {
|
||||
if (Object.prototype.hasOwnProperty.call(assets, assetID)) {
|
||||
const asset = assets[assetID];
|
||||
if (asset.queue?.length > 0) {
|
||||
await processQueue(assetID);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// each new payload will start new processQueue calls, which may cause some extra calls
|
||||
// it's essentially saying "start processing this asset again, I added more data to the queue"
|
||||
for (const assetID in assets) {
|
||||
if (Object.prototype.hasOwnProperty.call(assets, assetID)) {
|
||||
const asset = assets[assetID];
|
||||
if (asset.queue?.length > 0) {
|
||||
await processQueue(assetID);
|
||||
}
|
||||
const processor = new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(payload: Protocol.Client.TransferAssetFlow[], _encoding, callback) {
|
||||
processAssetPayload(payload).then(
|
||||
() => {
|
||||
callback();
|
||||
},
|
||||
(err: Error) => {
|
||||
clearAllStallTimeouts();
|
||||
stream.destroy(err);
|
||||
callback(err);
|
||||
}
|
||||
}
|
||||
})
|
||||
.on('close', () => {
|
||||
);
|
||||
},
|
||||
final(callback) {
|
||||
pass.end();
|
||||
});
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
processor.on('error', (err) => {
|
||||
clearAllStallTimeouts();
|
||||
pass.destroy(err);
|
||||
});
|
||||
|
||||
stream.on('error', (err) => {
|
||||
clearAllStallTimeouts();
|
||||
processor.destroy(err);
|
||||
pass.destroy(err);
|
||||
});
|
||||
|
||||
stream.once('end', () => {
|
||||
clearAllStallTimeouts();
|
||||
});
|
||||
|
||||
stream.pipe(processor);
|
||||
|
||||
/**
|
||||
* Start processing the queue for a given assetID
|
||||
@@ -265,16 +354,21 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
// if this is an end chunk, close the asset stream
|
||||
if (data.action === 'end') {
|
||||
this.#reportInfo(`Ending asset stream for ${id}`);
|
||||
await closeAssetStream(id);
|
||||
await closeAssetStream(id, data.checksum);
|
||||
break; // Exit the loop after closing the stream
|
||||
}
|
||||
|
||||
// Save the current chunk
|
||||
await writeChunkToStream(id, data);
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (!assets[id]) {
|
||||
throw new Error(`No id matching ${id} for writeAssetChunk`);
|
||||
}
|
||||
clearStallTimeoutForAsset(id);
|
||||
if (error instanceof Error) {
|
||||
throw error;
|
||||
}
|
||||
throw new Error(`Unexpected error while processing asset chunk for "${id}"`);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -284,17 +378,23 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
*
|
||||
* Only check if the targeted asset exists, no other validation is done.
|
||||
*/
|
||||
const writeChunkToStream = async (id: string, data: unknown) => {
|
||||
const writeChunkToStream = async (id: string, item: QueueableAction) => {
|
||||
const asset = assets[id];
|
||||
|
||||
if (!asset) {
|
||||
throw new Error(`Failed to write asset chunk for "${id}". Asset not found.`);
|
||||
}
|
||||
|
||||
const rawBuffer = data as { type: 'Buffer'; data: Uint8Array };
|
||||
const chunk = Buffer.from(rawBuffer.data);
|
||||
if (item.action !== 'stream') {
|
||||
throw new Error(`Expected stream queue item for "${id}"`);
|
||||
}
|
||||
|
||||
await this.writeAsync(asset.stream, chunk);
|
||||
const chunk = decodeTransferAssetStreamItem(item);
|
||||
asset.checksumHash?.update(chunk);
|
||||
|
||||
await write(asset.stream, chunk);
|
||||
// Count slow draining as progress so backpressure on large chunks does not trip the stall timer
|
||||
resetTimeout(id);
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -302,22 +402,49 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
*
|
||||
* It deletes the stream for the asset upon successful closure.
|
||||
*/
|
||||
const closeAssetStream = async (id: string) => {
|
||||
const closeAssetStream = async (
|
||||
id: string,
|
||||
checksum?: { algorithm: 'sha256'; value: string }
|
||||
) => {
|
||||
if (!assets[id]) {
|
||||
throw new Error(`Failed to close asset "${id}". Asset not found.`);
|
||||
}
|
||||
|
||||
assets[id].status = 'closed';
|
||||
const asset = assets[id];
|
||||
// The queue processes stream chunks before `end`; the last `writeChunkToStream` calls
|
||||
// `resetTimeout` after the `end` chunk already cleared the timer — clear again before closing.
|
||||
clearStallTimeoutForAsset(id);
|
||||
|
||||
if (this.#checksumsEnabled) {
|
||||
if (!checksum) {
|
||||
throw new ProviderTransferError(
|
||||
`Asset ${id} is missing checksum in transfer end payload`
|
||||
);
|
||||
}
|
||||
if (checksum.algorithm !== 'sha256') {
|
||||
throw new ProviderTransferError(
|
||||
`Asset ${id} checksum algorithm "${checksum.algorithm}" is not supported`
|
||||
);
|
||||
}
|
||||
const actual = asset.checksumHash?.digest('hex');
|
||||
if (!actual || actual !== checksum.value) {
|
||||
throw new ProviderTransferError(
|
||||
`Checksum mismatch for asset "${id}" (expected ${checksum.value}, got ${actual ?? 'none'})`
|
||||
);
|
||||
}
|
||||
}
|
||||
asset.status = 'closed';
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const { stream } = assets[id];
|
||||
const { stream } = asset;
|
||||
|
||||
stream
|
||||
.on('close', () => {
|
||||
delete assets[id];
|
||||
resolve();
|
||||
})
|
||||
.on('error', (e) => {
|
||||
assets[id].status = 'errored';
|
||||
delete assets[id];
|
||||
reject(new Error(`Failed to close asset "${id}". Asset stream error: ${e.toString()}`));
|
||||
})
|
||||
.end();
|
||||
@@ -352,15 +479,25 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
}
|
||||
|
||||
async initTransfer(): Promise<string> {
|
||||
const wantsChecksums = this.options.verifyChecksums === true;
|
||||
const query = this.dispatcher?.dispatchCommand({
|
||||
command: 'init',
|
||||
...(wantsChecksums ? { params: { transfer: 'pull', checksums: true } } : {}),
|
||||
});
|
||||
|
||||
const res = (await query) as Server.Payload<Server.InitMessage>;
|
||||
const res = (await query) as
|
||||
| (Server.Payload<Server.InitMessage> & { checksums?: boolean })
|
||||
| null;
|
||||
|
||||
if (!res?.transferID) {
|
||||
throw new ProviderTransferError('Init failed, invalid response from the server');
|
||||
}
|
||||
this.#checksumsEnabled = wantsChecksums && res.checksums === true;
|
||||
if (wantsChecksums && res.checksums !== true) {
|
||||
this.#reportWarning(
|
||||
'[Data transfer][pull] Checksums were requested but the remote does not support checksum negotiation; continuing without checksum validation.'
|
||||
);
|
||||
}
|
||||
|
||||
return res.transferID;
|
||||
}
|
||||
@@ -376,6 +513,18 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
});
|
||||
}
|
||||
|
||||
/** Reports a warning diagnostic (`kind: 'warning'`). Consumers (e.g. CLI) choose log levels and routing. */
|
||||
#reportWarning(message: string) {
|
||||
this.#diagnostics?.report({
|
||||
details: {
|
||||
createdAt: new Date(),
|
||||
message,
|
||||
origin: 'remote-source-provider',
|
||||
},
|
||||
kind: 'warning',
|
||||
});
|
||||
}
|
||||
|
||||
async bootstrap(diagnostics?: IDiagnosticReporter): Promise<void> {
|
||||
this.#diagnostics = diagnostics;
|
||||
const { url, auth } = this.options;
|
||||
@@ -386,6 +535,8 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
url.pathname
|
||||
)}${TRANSFER_PATH}/pull`;
|
||||
|
||||
this.#pullAssetStreamWireSampleLogged = false;
|
||||
|
||||
this.#reportInfo('establishing websocket connection');
|
||||
// No auth defined, trying public access for transfer
|
||||
if (!auth) {
|
||||
@@ -416,7 +567,7 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
this.dispatcher = createDispatcher(this.ws, retryMessageOptions, (message: string) =>
|
||||
this.#reportInfo(message)
|
||||
);
|
||||
this.#reportInfo('creating dispatcher');
|
||||
this.#reportInfo('created dispatcher');
|
||||
|
||||
this.#reportInfo('initialize transfer');
|
||||
const transferID = await this.initTransfer();
|
||||
@@ -448,9 +599,20 @@ class RemoteStrapiSourceProvider implements ISourceProvider {
|
||||
return schemas ?? null;
|
||||
}
|
||||
|
||||
async getStageTotals(stage: TransferStage): Promise<StageTotalsEstimate | null> {
|
||||
if (stage !== 'assets') {
|
||||
return null;
|
||||
}
|
||||
const cached = this.#cachedAssetsTotals;
|
||||
return cached ?? null;
|
||||
}
|
||||
|
||||
async #startStep<T extends Client.TransferPullStep>(step: T) {
|
||||
try {
|
||||
return await this.dispatcher?.dispatchTransferStep({ action: 'start', step });
|
||||
return await this.dispatcher?.dispatchTransferStep(
|
||||
{ action: 'start', step },
|
||||
step === 'assets' ? { retryOverrides: ASSETS_START_RETRY_OVERRIDES } : undefined
|
||||
);
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
return e;
|
||||
|
||||
@@ -11,20 +11,28 @@ import {
|
||||
ProviderErrorDetails,
|
||||
} from '../../errors/providers';
|
||||
import { IDiagnosticReporter } from '../../utils/diagnostic';
|
||||
import { stringifyTransferWebSocketPayload } from '../../utils/transfer-websocket-json';
|
||||
|
||||
interface IDispatcherState {
|
||||
transfer?: { kind: Client.TransferKind; id: string };
|
||||
}
|
||||
|
||||
export interface RetryMessageOptions {
|
||||
retryMessageMaxRetries: number;
|
||||
retryMessageTimeout: number;
|
||||
}
|
||||
|
||||
interface IDispatchOptions {
|
||||
attachTransfer?: boolean;
|
||||
/** Merged onto the dispatcher's default `retryMessageOptions` for this message only. */
|
||||
retryOverrides?: Partial<RetryMessageOptions>;
|
||||
}
|
||||
|
||||
type Dispatch<T> = Omit<T, 'transferID' | 'uuid'>;
|
||||
|
||||
export const createDispatcher = (
|
||||
ws: WebSocket,
|
||||
retryMessageOptions = {
|
||||
retryMessageOptions: RetryMessageOptions = {
|
||||
retryMessageMaxRetries: 5,
|
||||
retryMessageTimeout: 30000,
|
||||
},
|
||||
@@ -61,13 +69,18 @@ export const createDispatcher = (
|
||||
`dispatching message action:${messageToSend.action} ${messageToSend.kind === 'step' ? `step:${messageToSend.step}` : ''} uuid:${uuid} sent:${numberOfTimesMessageWasSent}`
|
||||
);
|
||||
}
|
||||
const stringifiedPayload = JSON.stringify(payload);
|
||||
const stringifiedPayload = stringifyTransferWebSocketPayload(
|
||||
payload as Record<string, unknown>
|
||||
);
|
||||
ws.send(stringifiedPayload, (error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
const { retryMessageMaxRetries, retryMessageTimeout } = retryMessageOptions;
|
||||
const { retryMessageMaxRetries, retryMessageTimeout } = {
|
||||
...retryMessageOptions,
|
||||
...options.retryOverrides,
|
||||
};
|
||||
const sendPeriodically = () => {
|
||||
if (numberOfTimesMessageWasSent <= retryMessageMaxRetries) {
|
||||
numberOfTimesMessageWasSent += 1;
|
||||
@@ -144,7 +157,8 @@ export const createDispatcher = (
|
||||
payload: {
|
||||
step: S;
|
||||
action: A;
|
||||
} & (A extends 'stream' ? { data: Client.GetTransferPushStreamData<S> } : unknown)
|
||||
} & (A extends 'stream' ? { data: Client.GetTransferPushStreamData<S> } : unknown),
|
||||
dispatchOptions?: Omit<IDispatchOptions, 'attachTransfer'>
|
||||
) => {
|
||||
const message: Dispatch<Client.TransferPushMessage> = {
|
||||
type: 'transfer',
|
||||
@@ -152,7 +166,9 @@ export const createDispatcher = (
|
||||
...payload,
|
||||
};
|
||||
|
||||
return dispatch<T>(message, { attachTransfer: true }) ?? Promise.resolve(null);
|
||||
return (
|
||||
dispatch<T>(message, { attachTransfer: true, ...dispatchOptions }) ?? Promise.resolve(null)
|
||||
);
|
||||
};
|
||||
|
||||
const setTransferProperties = (
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
import { Readable } from 'stream';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { randomUUID, createHash } from 'crypto';
|
||||
import type { Core } from '@strapi/types';
|
||||
|
||||
import { Handler } from './abstract';
|
||||
import { handlerControllerFactory, isDataTransferMessage } from './utils';
|
||||
import { createLocalStrapiSourceProvider, ILocalStrapiSourceProvider } from '../../providers';
|
||||
import {
|
||||
createTransferAssetStreamChunk,
|
||||
transferAssetStreamChunkByteLength,
|
||||
} from '../../../utils/transfer-asset-chunk';
|
||||
import {
|
||||
createLocalStrapiSourceProvider,
|
||||
estimateAssetTotals,
|
||||
ILocalStrapiSourceProvider,
|
||||
} from '../../providers';
|
||||
import { ProviderTransferError } from '../../../errors/providers';
|
||||
import type { IAsset, TransferStage, Protocol } from '../../../../types';
|
||||
import type { IAsset, StageTotalsEstimate, TransferStage, Protocol } from '../../../../types';
|
||||
import { Client } from '../../../../types/remote/protocol';
|
||||
|
||||
const TRANSFER_KIND = 'pull';
|
||||
@@ -18,6 +26,7 @@ export interface PullHandler extends Handler {
|
||||
provider?: ILocalStrapiSourceProvider;
|
||||
|
||||
streams?: { [stage in TransferStage]?: Readable };
|
||||
checksumsEnabled?: boolean;
|
||||
|
||||
assertValidTransferAction(action: string): asserts action is PullTransferAction;
|
||||
|
||||
@@ -43,6 +52,7 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
|
||||
proto.cleanup.call(this);
|
||||
|
||||
this.streams = {};
|
||||
this.checksumsEnabled = false;
|
||||
|
||||
delete this.provider;
|
||||
},
|
||||
@@ -205,11 +215,11 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
|
||||
batch = [];
|
||||
};
|
||||
|
||||
if (!stream) {
|
||||
throw new ProviderTransferError(`No available stream found for ${stage}`);
|
||||
}
|
||||
|
||||
try {
|
||||
if (!stream) {
|
||||
throw new ProviderTransferError(`No available stream found for ${stage}`);
|
||||
}
|
||||
|
||||
for await (const chunk of stream) {
|
||||
if (stage !== 'assets') {
|
||||
batch.push(chunk);
|
||||
@@ -247,10 +257,21 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
|
||||
|
||||
const flushUUID = randomUUID();
|
||||
|
||||
await this.createReadableStreamForStep(step);
|
||||
this.flush(step, flushUUID);
|
||||
let totals: StageTotalsEstimate | undefined;
|
||||
if (step === 'assets') {
|
||||
totals = await estimateAssetTotals(strapi as Core.Strapi);
|
||||
}
|
||||
|
||||
return { ok: true, id: flushUUID };
|
||||
await this.createReadableStreamForStep(step);
|
||||
Promise.resolve(this.flush(step, flushUUID)).catch((err: unknown) => {
|
||||
this.onError(err instanceof Error ? err : new Error(String(err)));
|
||||
});
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
id: flushUUID,
|
||||
...(totals !== undefined ? { totals } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
if (action === 'end') {
|
||||
@@ -276,12 +297,10 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
|
||||
assets: () => {
|
||||
const assets = this.provider?.createAssetsReadStream();
|
||||
let batch: Protocol.Client.TransferAssetFlow[] = [];
|
||||
const checksumsEnabled = this.checksumsEnabled === true;
|
||||
|
||||
const batchLength = () => {
|
||||
return batch.reduce(
|
||||
(acc, chunk) => (chunk.action === 'stream' ? acc + chunk.data.byteLength : acc),
|
||||
0
|
||||
);
|
||||
return batch.reduce((acc, chunk) => acc + transferAssetStreamChunkByteLength(chunk), 0);
|
||||
};
|
||||
|
||||
const BATCH_MAX_SIZE = 1024 * 1024; // 1MB
|
||||
@@ -298,19 +317,21 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
|
||||
async function* generator(stream: Readable) {
|
||||
let hasStarted = false;
|
||||
let assetID = '';
|
||||
let assetChecksum: ReturnType<typeof createHash> | undefined;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const { stream: assetStream, ...assetData } = chunk as IAsset;
|
||||
if (!hasStarted) {
|
||||
assetID = randomUUID();
|
||||
assetChecksum = checksumsEnabled ? createHash('sha256') : undefined;
|
||||
// Start the transfer of a new asset
|
||||
batch.push({ action: 'start', assetID, data: assetData });
|
||||
hasStarted = true;
|
||||
}
|
||||
|
||||
for await (const assetChunk of assetStream) {
|
||||
// Add the asset data to the batch
|
||||
batch.push({ action: 'stream', assetID, data: assetChunk });
|
||||
assetChecksum?.update(assetChunk);
|
||||
batch.push(createTransferAssetStreamChunk(assetID, assetChunk));
|
||||
|
||||
// if the batch size is bigger than BATCH_MAX_SIZE stream the batch
|
||||
if (batchLength() >= BATCH_MAX_SIZE) {
|
||||
@@ -321,7 +342,13 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
|
||||
|
||||
// All the asset data has been streamed and gets ready for the next one
|
||||
hasStarted = false;
|
||||
batch.push({ action: 'end', assetID });
|
||||
batch.push({
|
||||
action: 'end',
|
||||
assetID,
|
||||
...(assetChecksum
|
||||
? { checksum: { algorithm: 'sha256' as const, value: assetChecksum.digest('hex') } }
|
||||
: {}),
|
||||
});
|
||||
yield batch;
|
||||
batch = [];
|
||||
}
|
||||
@@ -343,7 +370,7 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
|
||||
},
|
||||
|
||||
// Commands
|
||||
async init(this: PullHandler) {
|
||||
async init(this: PullHandler, params?: Protocol.Client.GetCommandParams<'init'>) {
|
||||
if (this.transferID || this.provider) {
|
||||
throw new Error('Transfer already in progress');
|
||||
}
|
||||
@@ -351,6 +378,7 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
|
||||
|
||||
this.transferID = randomUUID();
|
||||
this.startedAt = Date.now();
|
||||
this.checksumsEnabled = params?.checksums === true;
|
||||
|
||||
this.streams = {};
|
||||
|
||||
@@ -359,7 +387,7 @@ export const createPullController = handlerControllerFactory<Partial<PullHandler
|
||||
getStrapi: () => strapi as Core.Strapi,
|
||||
});
|
||||
|
||||
return { transferID: this.transferID };
|
||||
return { transferID: this.transferID, checksums: true };
|
||||
},
|
||||
|
||||
async end(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { createHash, randomUUID, type Hash } from 'crypto';
|
||||
import { Writable, PassThrough } from 'stream';
|
||||
import type { Core } from '@strapi/types';
|
||||
|
||||
@@ -6,6 +6,8 @@ import type { TransferFlow, Step } from '../flows';
|
||||
import type { TransferStage, IAsset, Protocol } from '../../../../types';
|
||||
|
||||
import { ProviderTransferError } from '../../../errors/providers';
|
||||
import { write } from '../../../utils/writable-async-write';
|
||||
import { decodeTransferAssetStreamItem } from '../../../utils/transfer-asset-chunk';
|
||||
import { createLocalStrapiDestinationProvider } from '../../providers';
|
||||
import { createFlow, DEFAULT_TRANSFER_FLOW } from '../flows';
|
||||
import { Handler } from './abstract';
|
||||
@@ -42,13 +44,21 @@ export interface PushHandler extends Handler {
|
||||
/**
|
||||
* Holds all the transferred assets for the current transfer handler (one registry per connection)
|
||||
*/
|
||||
assets: { [filepath: string]: IAsset & { stream: PassThrough } };
|
||||
assets: { [assetID: string]: IAsset & { stream: PassThrough } };
|
||||
/** Incremental checksum state keyed by transfer asset ID (only populated when checksums are enabled). */
|
||||
assetChecksums?: { [assetID: string]: Hash };
|
||||
checksumsEnabled?: boolean;
|
||||
|
||||
/**
|
||||
* Ochestrate and manage the transfer messages' ordering
|
||||
* Orchestrate and manage the transfer messages' ordering
|
||||
*/
|
||||
flow?: TransferFlow;
|
||||
|
||||
/**
|
||||
* Interval for periodic destination memory logging during assets stage
|
||||
*/
|
||||
memoryLogInterval?: ReturnType<typeof setInterval>;
|
||||
|
||||
/**
|
||||
* Checks that the given action is a valid push transfer action
|
||||
*/
|
||||
@@ -105,18 +115,6 @@ export interface PushHandler extends Handler {
|
||||
assertValidStreamTransferStep(stage: TransferStage): void;
|
||||
}
|
||||
|
||||
const writeAsync = <T>(stream: Writable, data: T) => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
stream.write(data, (error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
}
|
||||
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
export const createPushController = handlerControllerFactory<Partial<PushHandler>>((proto) => ({
|
||||
isTransferStarted(this: PushHandler) {
|
||||
return proto.isTransferStarted.call(this) && this.provider !== undefined;
|
||||
@@ -146,16 +144,26 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
|
||||
});
|
||||
},
|
||||
cleanup(this: PushHandler) {
|
||||
if (this.memoryLogInterval) {
|
||||
clearInterval(this.memoryLogInterval);
|
||||
delete this.memoryLogInterval;
|
||||
}
|
||||
proto.cleanup.call(this);
|
||||
|
||||
this.streams = {};
|
||||
this.assets = {};
|
||||
this.assetChecksums = {};
|
||||
this.checksumsEnabled = false;
|
||||
|
||||
delete this.flow;
|
||||
delete this.provider;
|
||||
},
|
||||
|
||||
teardown(this: PushHandler) {
|
||||
if (this.memoryLogInterval) {
|
||||
clearInterval(this.memoryLogInterval);
|
||||
delete this.memoryLogInterval;
|
||||
}
|
||||
if (this.provider) {
|
||||
this.provider.rollback();
|
||||
}
|
||||
@@ -338,6 +346,21 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
|
||||
|
||||
this.stats[stage] = { started: 0, finished: 0 };
|
||||
|
||||
if (stage === 'assets') {
|
||||
strapi.log.debug(
|
||||
'[Transfer destination] Assets stage started; sampling memory usage every 5s until stage end'
|
||||
);
|
||||
this.memoryLogInterval = setInterval(() => {
|
||||
const mem = process.memoryUsage();
|
||||
const stats = this.stats?.assets;
|
||||
const rssMb = (mem.rss / 1024 / 1024).toFixed(1);
|
||||
const heapMb = (mem.heapUsed / 1024 / 1024).toFixed(1);
|
||||
strapi.log.debug(
|
||||
`[Transfer destination] memory RSS=${rssMb}MB heapUsed=${heapMb}MB | assets started=${stats?.started ?? 0} finished=${stats?.finished ?? 0}`
|
||||
);
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
@@ -356,17 +379,20 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
|
||||
return this.streamAsset(msg.data);
|
||||
}
|
||||
|
||||
// For all other steps
|
||||
await Promise.all(
|
||||
msg.data.map(async (item) => {
|
||||
this.stats[stage].started += 1;
|
||||
await writeAsync(stream, item);
|
||||
this.stats[stage].finished += 1;
|
||||
})
|
||||
);
|
||||
// One objectMode Writable: do not overlap writes.
|
||||
for (const item of msg.data) {
|
||||
this.stats[stage].started += 1;
|
||||
await write(stream, item);
|
||||
this.stats[stage].finished += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (msg.action === 'end') {
|
||||
if (stage === 'assets' && this.memoryLogInterval) {
|
||||
clearInterval(this.memoryLogInterval);
|
||||
delete this.memoryLogInterval;
|
||||
strapi.log.debug('[Transfer destination] Assets stage ended, stopped memory log');
|
||||
}
|
||||
this.unlockTransferStep(stage);
|
||||
const stream = this.streams?.[stage];
|
||||
|
||||
@@ -424,29 +450,66 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
|
||||
if (action === 'start') {
|
||||
this.stats.assets.started += 1;
|
||||
this.assets[assetID] = { ...item.data, stream: new PassThrough() };
|
||||
writeAsync(assetsStream, this.assets[assetID]);
|
||||
}
|
||||
if (this.checksumsEnabled) {
|
||||
this.assetChecksums ??= {};
|
||||
this.assetChecksums[assetID] = createHash('sha256');
|
||||
}
|
||||
const filename = item.data?.filename ?? assetID;
|
||||
strapi.log.debug(
|
||||
`[Transfer destination] Asset start #${this.stats.assets.started} id=${assetID} filename=${filename}`
|
||||
);
|
||||
// Wait for the assets stage to accept this row (same pattern as remote-source).
|
||||
await write(assetsStream, this.assets[assetID]);
|
||||
} else if (action === 'stream' || action === 'end') {
|
||||
if (!this.assets[assetID]) {
|
||||
throw new ProviderTransferError(
|
||||
`No asset "${assetID}" for ${action} action; send start before stream/end`
|
||||
);
|
||||
}
|
||||
|
||||
if (action === 'stream') {
|
||||
// The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
|
||||
// We need to transform it back into a Buffer instance
|
||||
const rawBuffer = item.data as unknown as { type: 'Buffer'; data: Uint8Array };
|
||||
const chunk = Buffer.from(rawBuffer.data);
|
||||
await writeAsync(this.assets[assetID].stream, chunk);
|
||||
}
|
||||
|
||||
if (action === 'end') {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const { stream: assetStream } = this.assets[assetID];
|
||||
assetStream
|
||||
.on('close', () => {
|
||||
this.stats.assets.finished += 1;
|
||||
delete this.assets[assetID];
|
||||
resolve();
|
||||
})
|
||||
.on('error', reject)
|
||||
.end();
|
||||
});
|
||||
if (action === 'stream') {
|
||||
const chunk = decodeTransferAssetStreamItem(item);
|
||||
this.assetChecksums?.[assetID]?.update(chunk);
|
||||
await write(this.assets[assetID].stream, chunk);
|
||||
} else {
|
||||
if (this.checksumsEnabled) {
|
||||
if (!item.checksum) {
|
||||
throw new ProviderTransferError(`Missing checksum for asset "${assetID}"`);
|
||||
}
|
||||
if (item.checksum.algorithm !== 'sha256') {
|
||||
throw new ProviderTransferError(
|
||||
`Unsupported checksum algorithm "${item.checksum.algorithm}" for asset ${assetID}`
|
||||
);
|
||||
}
|
||||
const checksum = this.assetChecksums?.[assetID]?.digest('hex');
|
||||
if (!checksum || checksum !== item.checksum.value) {
|
||||
throw new ProviderTransferError(
|
||||
`Checksum mismatch for asset "${assetID}" (expected ${item.checksum.value}, got ${checksum ?? 'none'})`
|
||||
);
|
||||
}
|
||||
}
|
||||
if (this.assetChecksums?.[assetID]) {
|
||||
delete this.assetChecksums[assetID];
|
||||
}
|
||||
strapi.log.debug(
|
||||
`[Transfer destination] Asset end id=${assetID} (finished=${this.stats.assets.finished + 1}/${this.stats.assets.started})`
|
||||
);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const { stream: assetStream } = this.assets[assetID];
|
||||
assetStream
|
||||
.on('close', () => {
|
||||
this.stats.assets.finished += 1;
|
||||
delete this.assets[assetID];
|
||||
resolve();
|
||||
})
|
||||
.on('error', reject)
|
||||
.end();
|
||||
});
|
||||
}
|
||||
} else {
|
||||
throw new ProviderTransferError(
|
||||
`Invalid asset flow action: ${String((item as { action?: unknown }).action)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -476,6 +539,8 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
|
||||
this.startedAt = Date.now();
|
||||
|
||||
this.assets = {};
|
||||
this.assetChecksums = {};
|
||||
this.checksumsEnabled = params?.checksums === true;
|
||||
this.streams = {};
|
||||
this.stats = {
|
||||
assets: { started: 0, finished: 0 },
|
||||
@@ -487,7 +552,8 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
|
||||
this.flow = createFlow(DEFAULT_TRANSFER_FLOW);
|
||||
|
||||
this.provider = createLocalStrapiDestinationProvider({
|
||||
...params.options,
|
||||
strategy: params?.options?.strategy ?? 'restore',
|
||||
restore: params?.options?.restore ?? {},
|
||||
autoDestroy: false,
|
||||
getStrapi: () => strapi as Core.Strapi,
|
||||
});
|
||||
@@ -497,7 +563,7 @@ export const createPushController = handlerControllerFactory<Partial<PushHandler
|
||||
strapi.log.warn(message);
|
||||
};
|
||||
|
||||
return { transferID: this.transferID };
|
||||
return { transferID: this.transferID, checksums: true };
|
||||
},
|
||||
|
||||
async status(this: PushHandler) {
|
||||
|
||||
@@ -10,6 +10,7 @@ import { ProviderError, ProviderTransferError } from '../../../errors/providers'
|
||||
import { VALID_TRANSFER_COMMANDS, ValidTransferCommand } from './constants';
|
||||
import { TransferMethod } from '../constants';
|
||||
import { createDiagnosticReporter } from '../../../utils/diagnostic';
|
||||
import { stringifyTransferWebSocketPayload } from '../../../utils/transfer-websocket-json';
|
||||
|
||||
type WSCallback = (client: WebSocket, request: IncomingMessage) => void;
|
||||
|
||||
@@ -245,7 +246,7 @@ export const handlerControllerFactory =
|
||||
details = e.details;
|
||||
}
|
||||
|
||||
const payload = JSON.stringify({
|
||||
const envelope: Record<string, unknown> = {
|
||||
uuid,
|
||||
data: data ?? null,
|
||||
error: e
|
||||
@@ -255,7 +256,9 @@ export const handlerControllerFactory =
|
||||
details,
|
||||
}
|
||||
: null,
|
||||
});
|
||||
};
|
||||
|
||||
const payload = stringifyTransferWebSocketPayload(envelope);
|
||||
|
||||
this.send(payload, (error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
@@ -268,7 +271,7 @@ export const handlerControllerFactory =
|
||||
return new Promise((resolve, reject) => {
|
||||
const uuid = randomUUID();
|
||||
|
||||
const payload = JSON.stringify({ uuid, data: message });
|
||||
const payload = stringifyTransferWebSocketPayload({ uuid, data: message });
|
||||
|
||||
this.send(payload, (error) => {
|
||||
if (error) {
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
import { Readable, Writable } from 'stream';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { filter, map, collect } from '../stream';
|
||||
|
||||
describe('stream utils', () => {
|
||||
describe('filter', () => {
|
||||
test('propagates backpressure: source pauses when downstream is slow', async () => {
|
||||
const itemCount = 20;
|
||||
const data = Array.from({ length: itemCount }, (_, i) => ({ id: i }));
|
||||
let sourcePaused = false;
|
||||
|
||||
const source = Readable.from(data, { objectMode: true });
|
||||
const originalPause = source.pause.bind(source);
|
||||
source.pause = function () {
|
||||
sourcePaused = true;
|
||||
return originalPause();
|
||||
};
|
||||
|
||||
const slowConsumer = new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(chunk, _encoding, callback) {
|
||||
setTimeout(callback, 5);
|
||||
},
|
||||
});
|
||||
|
||||
const filtered = filter<{ id: number }>((x) => x.id % 2 === 0);
|
||||
await pipeline(source, filtered, slowConsumer);
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
});
|
||||
|
||||
test('does not retain references to discarded chunks', async () => {
|
||||
const data = Array.from({ length: 100 }, (_, i) => ({ id: i }));
|
||||
const collected: Array<{ id: number }> = [];
|
||||
|
||||
const source = Readable.from(data, { objectMode: true });
|
||||
const filtered = filter<{ id: number }>((x) => x.id % 2 === 0);
|
||||
const writer = new Writable({
|
||||
objectMode: true,
|
||||
write(chunk, _encoding, callback) {
|
||||
collected.push(chunk);
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
await pipeline(source, filtered, writer);
|
||||
expect(collected).toHaveLength(50);
|
||||
expect(collected.map((x) => x.id)).toEqual(
|
||||
data.filter((_, i) => i % 2 === 0).map((x) => x.id)
|
||||
);
|
||||
});
|
||||
|
||||
test('stream is destroyed after pipeline completes', async () => {
|
||||
const source = Readable.from([1, 2, 3], { objectMode: true });
|
||||
const filtered = filter<number>(() => true);
|
||||
const dest = new Writable({
|
||||
objectMode: true,
|
||||
write(chunk, _enc, cb) {
|
||||
cb();
|
||||
},
|
||||
});
|
||||
await pipeline(source, filtered, dest);
|
||||
expect(source.destroyed).toBe(true);
|
||||
expect(filtered.destroyed).toBe(true);
|
||||
expect(dest.destroyed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('map', () => {
|
||||
test('propagates backpressure: source pauses when downstream is slow', async () => {
|
||||
const itemCount = 20;
|
||||
const data = Array.from({ length: itemCount }, (_, i) => i);
|
||||
let sourcePaused = false;
|
||||
|
||||
const source = Readable.from(data, { objectMode: true });
|
||||
const originalPause = source.pause.bind(source);
|
||||
source.pause = function () {
|
||||
sourcePaused = true;
|
||||
return originalPause();
|
||||
};
|
||||
|
||||
const slowConsumer = new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(_chunk, _encoding, callback) {
|
||||
setTimeout(callback, 5);
|
||||
},
|
||||
});
|
||||
|
||||
const mapped = map<number, number>((x) => x * 2);
|
||||
await pipeline(source, mapped, slowConsumer);
|
||||
|
||||
expect(sourcePaused).toBe(true);
|
||||
});
|
||||
|
||||
test('does not retain references to all chunks', async () => {
|
||||
const data = Array.from({ length: 100 }, (_, i) => i);
|
||||
const collected: number[] = [];
|
||||
|
||||
const source = Readable.from(data, { objectMode: true });
|
||||
const mapped = map<number, number>((x) => x + 1);
|
||||
const writer = new Writable({
|
||||
objectMode: true,
|
||||
write(chunk, _encoding, callback) {
|
||||
collected.push(chunk);
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
await pipeline(source, mapped, writer);
|
||||
expect(collected).toHaveLength(100);
|
||||
expect(collected).toEqual(data.map((x) => x + 1));
|
||||
});
|
||||
|
||||
test('stream is destroyed after pipeline completes', async () => {
|
||||
const source = Readable.from([1, 2, 3], { objectMode: true });
|
||||
const mapped = map<number, number>((x) => x);
|
||||
const dest = new Writable({
|
||||
objectMode: true,
|
||||
write(chunk, _enc, cb) {
|
||||
cb();
|
||||
},
|
||||
});
|
||||
await pipeline(source, mapped, dest);
|
||||
expect(source.destroyed).toBe(true);
|
||||
expect(mapped.destroyed).toBe(true);
|
||||
expect(dest.destroyed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('collect', () => {
|
||||
test('destroys stream when options.destroy is true (default)', async () => {
|
||||
const source = Readable.from([1, 2, 3], { objectMode: true });
|
||||
const result = await collect<number>(source);
|
||||
expect(result).toEqual([1, 2, 3]);
|
||||
expect(source.destroyed).toBe(true);
|
||||
});
|
||||
|
||||
test('with destroy false still resolves with correct data', async () => {
|
||||
const source = Readable.from([1, 2, 3], { objectMode: true });
|
||||
const result = await collect<number>(source, { destroy: false });
|
||||
expect(result).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
test('resolves on close with chunks received so far', async () => {
|
||||
const source = new Readable({
|
||||
objectMode: true,
|
||||
read() {
|
||||
this.push(1);
|
||||
this.push(2);
|
||||
this.destroy();
|
||||
},
|
||||
});
|
||||
const result = await collect<number>(source);
|
||||
expect(result).toEqual([1, 2]);
|
||||
});
|
||||
|
||||
test('rejects on stream error', async () => {
|
||||
const source = new Readable({
|
||||
objectMode: true,
|
||||
read() {
|
||||
this.destroy(new Error('test error'));
|
||||
},
|
||||
});
|
||||
await expect(collect(source)).rejects.toThrow('test error');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,139 @@
|
||||
import {
|
||||
createTransferAssetStreamChunk,
|
||||
decodeTransferAssetStreamData,
|
||||
decodeTransferAssetStreamItem,
|
||||
transferAssetStreamChunkByteLength,
|
||||
} from '../transfer-asset-chunk';
|
||||
import { replacerForTransferWebSocket } from '../transfer-websocket-json';
|
||||
|
||||
describe('transfer-asset-chunk', () => {
|
||||
const bytes = Buffer.from([0xde, 0xad, 0xbe, 0xef]);
|
||||
|
||||
test('createTransferAssetStreamChunk + decodeTransferAssetStreamItem round-trip', () => {
|
||||
const item = createTransferAssetStreamChunk('asset-1', bytes);
|
||||
expect(item).toEqual(
|
||||
expect.objectContaining({ action: 'stream', assetID: 'asset-1', encoding: 'base64' })
|
||||
);
|
||||
expect(decodeTransferAssetStreamItem(item)).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('decodeTransferAssetStreamItem: legacy Buffer JSON item', () => {
|
||||
const item = {
|
||||
action: 'stream' as const,
|
||||
assetID: 'a',
|
||||
data: { type: 'Buffer' as const, data: Array.from(bytes) },
|
||||
};
|
||||
expect(decodeTransferAssetStreamItem(item)).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('decodeTransferAssetStreamData: explicit encoding base64', () => {
|
||||
expect(decodeTransferAssetStreamData(bytes.toString('base64'), 'base64')).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('decodeTransferAssetStreamData: legacy Buffer JSON shape (client push / old pull)', () => {
|
||||
const legacy = { type: 'Buffer' as const, data: Array.from(bytes) };
|
||||
expect(decodeTransferAssetStreamData(legacy)).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('decodeTransferAssetStreamData: Buffer instance (in-process)', () => {
|
||||
expect(decodeTransferAssetStreamData(Buffer.from(bytes))).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('decodeTransferAssetStreamData: base64 string without encoding flag', () => {
|
||||
const b64 = bytes.toString('base64');
|
||||
expect(decodeTransferAssetStreamData(b64)).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('wire compatibility: Buffer property stringifies as legacy JSON (Node toJSON, not replacer)', () => {
|
||||
const item = { action: 'stream' as const, assetID: 'a', data: Buffer.from(bytes) };
|
||||
const wire = JSON.parse(JSON.stringify(item, replacerForTransferWebSocket)) as {
|
||||
data: unknown;
|
||||
};
|
||||
expect(wire.data).toEqual(expect.objectContaining({ type: 'Buffer' }));
|
||||
expect(decodeTransferAssetStreamData(wire.data)).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('wire compatibility: Uint8Array → WS replacer → base64 string → decode', () => {
|
||||
const item = { action: 'stream' as const, assetID: 'a', data: new Uint8Array(bytes) };
|
||||
const wire = JSON.parse(JSON.stringify(item, replacerForTransferWebSocket)) as {
|
||||
data: unknown;
|
||||
};
|
||||
expect(typeof wire.data).toBe('string');
|
||||
expect(decodeTransferAssetStreamData(wire.data)).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('wire compatibility: stream item legacy JSON shape survives parse unchanged', () => {
|
||||
const item = {
|
||||
action: 'stream' as const,
|
||||
assetID: 'a',
|
||||
data: { type: 'Buffer' as const, data: Array.from(bytes) },
|
||||
};
|
||||
const wire = JSON.parse(JSON.stringify(item)) as { data: unknown };
|
||||
expect(decodeTransferAssetStreamData(wire.data)).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('wire compatibility: explicit pull shape encoding+string', () => {
|
||||
const item = {
|
||||
action: 'stream' as const,
|
||||
assetID: 'a',
|
||||
encoding: 'base64' as const,
|
||||
data: bytes.toString('base64'),
|
||||
};
|
||||
const wire = JSON.parse(JSON.stringify(item)) as typeof item;
|
||||
expect(decodeTransferAssetStreamData(wire.data, wire.encoding)).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('transferAssetStreamChunkByteLength: base64 string (with or without encoding flag)', () => {
|
||||
const b64 = bytes.toString('base64');
|
||||
expect(
|
||||
transferAssetStreamChunkByteLength({
|
||||
action: 'stream',
|
||||
encoding: 'base64',
|
||||
data: b64,
|
||||
})
|
||||
).toBe(Math.floor((b64.length * 3) / 4));
|
||||
|
||||
expect(
|
||||
transferAssetStreamChunkByteLength({
|
||||
action: 'stream',
|
||||
data: b64,
|
||||
})
|
||||
).toBe(Math.floor((b64.length * 3) / 4));
|
||||
});
|
||||
|
||||
test('transferAssetStreamChunkByteLength: Buffer and non-stream', () => {
|
||||
expect(
|
||||
transferAssetStreamChunkByteLength({
|
||||
action: 'stream',
|
||||
data: bytes,
|
||||
})
|
||||
).toBe(4);
|
||||
|
||||
expect(transferAssetStreamChunkByteLength({ action: 'end' })).toBe(0);
|
||||
});
|
||||
|
||||
test('decodeTransferAssetStreamData: encoding base64 with legacy object falls back (no throw)', () => {
|
||||
const legacy = { type: 'Buffer' as const, data: Array.from(bytes) };
|
||||
expect(decodeTransferAssetStreamData(legacy, 'base64')).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('decodeTransferAssetStreamItem: encoding base64 with legacy payload', () => {
|
||||
const item = {
|
||||
action: 'stream' as const,
|
||||
assetID: 'a',
|
||||
encoding: 'base64' as const,
|
||||
data: { type: 'Buffer' as const, data: Array.from(bytes) },
|
||||
};
|
||||
expect(decodeTransferAssetStreamItem(item)).toEqual(bytes);
|
||||
});
|
||||
|
||||
test('transferAssetStreamChunkByteLength: legacy Buffer JSON shape', () => {
|
||||
const legacy = { type: 'Buffer' as const, data: Array.from(bytes) };
|
||||
expect(
|
||||
transferAssetStreamChunkByteLength({
|
||||
action: 'stream',
|
||||
data: legacy,
|
||||
})
|
||||
).toBe(bytes.length);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
import {
|
||||
replacerForTransferWebSocket,
|
||||
stringifyTransferWebSocketPayload,
|
||||
} from '../transfer-websocket-json';
|
||||
|
||||
describe('stringifyTransferWebSocketPayload', () => {
|
||||
test('raw JSON.stringify throws on BigInt; wire helper encodes nested bigint metadata as decimal strings', () => {
|
||||
expect(() => JSON.stringify({ size: 9007199254740993n })).toThrow();
|
||||
|
||||
const s = stringifyTransferWebSocketPayload({
|
||||
uuid: 'u',
|
||||
type: 'transfer',
|
||||
kind: 'step',
|
||||
action: 'stream',
|
||||
step: 'assets',
|
||||
data: [
|
||||
{
|
||||
action: 'start',
|
||||
assetID: '1',
|
||||
metadata: { size: 9007199254740993n, name: 'blob.bin' },
|
||||
},
|
||||
],
|
||||
} as Record<string, unknown>);
|
||||
|
||||
expect(s).toContain('"size":"9007199254740993"');
|
||||
expect(s).toContain('"name":"blob.bin"');
|
||||
});
|
||||
|
||||
test('circular nested data throws TypeError with context (peer cannot decode cycles)', () => {
|
||||
const cyclic: Record<string, unknown> = { tag: 'x' };
|
||||
cyclic.self = cyclic;
|
||||
|
||||
expect(() =>
|
||||
stringifyTransferWebSocketPayload({
|
||||
uuid: 'u',
|
||||
type: 'transfer',
|
||||
data: cyclic,
|
||||
} as Record<string, unknown>)
|
||||
).toThrow(TypeError);
|
||||
|
||||
expect(() =>
|
||||
stringifyTransferWebSocketPayload({
|
||||
uuid: 'u',
|
||||
type: 'transfer',
|
||||
data: cyclic,
|
||||
} as Record<string, unknown>)
|
||||
).toThrow(/could not be serialized to JSON/);
|
||||
});
|
||||
|
||||
test('upload-like nested metadata (unicode, null, empty object, deep keys) serializes', () => {
|
||||
const s = stringifyTransferWebSocketPayload({
|
||||
uuid: 'u',
|
||||
type: 'transfer',
|
||||
kind: 'step',
|
||||
action: 'stream',
|
||||
step: 'assets',
|
||||
data: {
|
||||
action: 'start',
|
||||
assetID: 'file-48',
|
||||
metadata: {
|
||||
name: 'café 文件 🗂️.bin',
|
||||
caption: null,
|
||||
alternativeText: '',
|
||||
provider_metadata: { foo: { bar: [1, 2, 3] } },
|
||||
},
|
||||
},
|
||||
} as Record<string, unknown>);
|
||||
|
||||
expect(typeof s).toBe('string');
|
||||
const roundTrip = JSON.parse(s) as { data: { metadata: { name: string } } };
|
||||
expect(roundTrip.data.metadata.name).toContain('café');
|
||||
});
|
||||
|
||||
test('enumerable toJSON on the root object makes raw JSON.stringify return undefined (ws.send would throw)', () => {
|
||||
const evil = {
|
||||
type: 'transfer',
|
||||
kind: 'step',
|
||||
action: 'stream',
|
||||
step: 'assets',
|
||||
data: [],
|
||||
toJSON() {
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
const payload = { ...evil, uuid: 'test-uuid' };
|
||||
expect(JSON.stringify(payload, replacerForTransferWebSocket)).toBeUndefined();
|
||||
});
|
||||
|
||||
test('stringifyTransferWebSocketPayload strips enumerable toJSON so serialization succeeds (fixes ws.send(undefined))', () => {
|
||||
const evil = {
|
||||
type: 'transfer',
|
||||
kind: 'step',
|
||||
action: 'stream',
|
||||
step: 'assets',
|
||||
data: [],
|
||||
toJSON() {
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
const payload = { ...evil, uuid: 'test-uuid' } as Record<string, unknown>;
|
||||
const s = stringifyTransferWebSocketPayload(payload);
|
||||
expect(typeof s).toBe('string');
|
||||
expect(s).toContain('"uuid":"test-uuid"');
|
||||
expect(s).toContain('"type":"transfer"');
|
||||
});
|
||||
|
||||
test('replacer encodes Uint8Array values as base64 strings (nested Buffer may use Buffer.toJSON before replacer)', () => {
|
||||
const payload = {
|
||||
uuid: 'u',
|
||||
raw: new Uint8Array([0xde, 0xad]),
|
||||
} as Record<string, unknown>;
|
||||
const s = stringifyTransferWebSocketPayload(payload);
|
||||
expect(s).toContain('3q0=');
|
||||
});
|
||||
|
||||
test('nested plain objects do not need root toJSON strip; only root spread could copy enumerable toJSON', () => {
|
||||
const inner = {
|
||||
id: 48,
|
||||
formats: { large: { url: '/u/x' } },
|
||||
toJSON() {
|
||||
return { id: 48, note: 'orm-shaped' };
|
||||
},
|
||||
};
|
||||
const s = stringifyTransferWebSocketPayload({
|
||||
uuid: 'u',
|
||||
type: 'transfer',
|
||||
data: { file: inner },
|
||||
} as Record<string, unknown>);
|
||||
|
||||
expect(typeof s).toBe('string');
|
||||
expect(s).toContain('"note":"orm-shaped"');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
import { Writable } from 'stream';
|
||||
|
||||
import { write } from '../writable-async-write';
|
||||
|
||||
describe('writable-async-write (write)', () => {
|
||||
const created: Writable[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
while (created.length) {
|
||||
created.pop()?.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('waits for drain when write returns false (slow consumer, low highWaterMark)', async () => {
|
||||
let drainCount = 0;
|
||||
const dest = new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(_chunk, _encoding, callback) {
|
||||
setTimeout(callback, 2);
|
||||
},
|
||||
});
|
||||
created.push(dest);
|
||||
dest.on('drain', () => {
|
||||
drainCount += 1;
|
||||
});
|
||||
|
||||
for (let i = 0; i < 12; i += 1) {
|
||||
await write(dest, { n: i });
|
||||
}
|
||||
|
||||
expect(drainCount).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('resolves when write returns true (room in buffer)', async () => {
|
||||
const dest = new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 100,
|
||||
write(_chunk, _encoding, callback) {
|
||||
callback();
|
||||
},
|
||||
});
|
||||
created.push(dest);
|
||||
|
||||
await expect(write(dest, { x: 1 })).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test('rejects when write callback receives an error', async () => {
|
||||
const dest = new Writable({
|
||||
objectMode: true,
|
||||
write(_chunk, _encoding, callback) {
|
||||
callback(new Error('sink error'));
|
||||
},
|
||||
});
|
||||
created.push(dest);
|
||||
|
||||
await expect(write(dest, { x: 1 })).rejects.toThrow('sink error');
|
||||
});
|
||||
|
||||
test('does not hang when writable is destroyed while awaiting drain (race with stream finished)', async () => {
|
||||
const dest = new Writable({
|
||||
objectMode: true,
|
||||
highWaterMark: 1,
|
||||
write(_chunk, _encoding, callback) {
|
||||
setTimeout(callback, 2);
|
||||
},
|
||||
});
|
||||
created.push(dest);
|
||||
|
||||
const run = async () => {
|
||||
for (let i = 0; i < 30; i += 1) {
|
||||
if (i === 8) {
|
||||
setImmediate(() => {
|
||||
dest.destroy(new Error('aborted'));
|
||||
});
|
||||
}
|
||||
await write(dest, { n: i });
|
||||
}
|
||||
};
|
||||
|
||||
await expect(run()).rejects.toThrow('aborted');
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,6 @@
|
||||
export * as encryption from './encryption';
|
||||
export * from './transfer-asset-chunk';
|
||||
export { write } from './writable-async-write';
|
||||
export * as stream from './stream';
|
||||
export * as json from './json';
|
||||
export * as schema from './schema';
|
||||
|
||||
@@ -57,16 +57,56 @@ export const collect = <T = unknown>(
|
||||
const chunks: T[] = [];
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream
|
||||
.on('close', () => resolve(chunks))
|
||||
.on('error', reject)
|
||||
.on('data', (chunk) => chunks.push(chunk))
|
||||
.on('end', () => {
|
||||
if (options.destroy) {
|
||||
stream.destroy();
|
||||
}
|
||||
let settled = false;
|
||||
|
||||
resolve(chunks);
|
||||
});
|
||||
const cleanup = () => {
|
||||
stream.removeListener('data', onData);
|
||||
stream.removeListener('end', onEnd);
|
||||
stream.removeListener('close', onClose);
|
||||
stream.removeListener('error', onError);
|
||||
};
|
||||
|
||||
const finishResolve = () => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
cleanup();
|
||||
resolve(chunks);
|
||||
};
|
||||
|
||||
const finishReject = (err: unknown) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(err);
|
||||
};
|
||||
|
||||
const onData = (chunk: T) => {
|
||||
chunks.push(chunk);
|
||||
};
|
||||
|
||||
const onEnd = () => {
|
||||
if (options.destroy) {
|
||||
stream.destroy();
|
||||
}
|
||||
finishResolve();
|
||||
};
|
||||
|
||||
const onClose = () => {
|
||||
// Handles streams that emit `close` without `end` (e.g. `destroy()` in `_read`).
|
||||
finishResolve();
|
||||
};
|
||||
|
||||
const onError = (err: Error) => {
|
||||
finishReject(err);
|
||||
};
|
||||
|
||||
stream.on('data', onData);
|
||||
stream.on('end', onEnd);
|
||||
stream.on('close', onClose);
|
||||
stream.on('error', onError);
|
||||
});
|
||||
};
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* Canonical **outbound** asset chunk for WebSocket JSON (push and pull).
|
||||
* Base64 string `data` keeps `JSON.parse` heap bounded vs `{ type: 'Buffer', data: [n,…] }`.
|
||||
*/
|
||||
export function createTransferAssetStreamChunk(
|
||||
assetID: string,
|
||||
chunk: Buffer | Uint8Array
|
||||
): { action: 'stream'; assetID: string; encoding: 'base64'; data: string } {
|
||||
if (chunk == null) {
|
||||
throw new TypeError(
|
||||
'Asset stream yielded a null/undefined chunk; refusing to encode (would trigger Buffer.from(undefined))'
|
||||
);
|
||||
}
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
return {
|
||||
action: 'stream',
|
||||
assetID,
|
||||
encoding: 'base64',
|
||||
data: buffer.toString('base64'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a stream item from `TransferAssetFlow` after `JSON.parse` (shared by push + pull handlers
|
||||
* and the remote source provider).
|
||||
*/
|
||||
export function decodeTransferAssetStreamItem(item: {
|
||||
action: 'stream';
|
||||
data: unknown;
|
||||
encoding?: 'base64';
|
||||
}): Buffer {
|
||||
return decodeTransferAssetStreamData(
|
||||
item.data,
|
||||
item.encoding === 'base64' ? 'base64' : undefined
|
||||
);
|
||||
}
|
||||
|
||||
const getLegacyBufferJsonData = (value: unknown): Uint8Array | readonly number[] | null => {
|
||||
if (!value || typeof value !== 'object' || !('type' in value)) {
|
||||
return null;
|
||||
}
|
||||
if ((value as { type: unknown }).type !== 'Buffer') {
|
||||
return null;
|
||||
}
|
||||
const raw = (value as { data?: unknown }).data;
|
||||
if (Array.isArray(raw) || ArrayBuffer.isView(raw)) {
|
||||
return raw as Uint8Array | readonly number[];
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Decode binary payload for `TransferAssetFlow` `action: 'stream'` after JSON.parse.
|
||||
*
|
||||
* Supported shapes (receivers should accept all of these):
|
||||
* - **String `data`:** preferred wire form (`createTransferAssetStreamChunk` / `encoding: 'base64'`).
|
||||
* - **`{ type: 'Buffer', data: number[] | TypedArray }`:** legacy `Buffer.toJSON()` from default
|
||||
* `JSON.stringify` (older clients/servers).
|
||||
* - **`Buffer` instance:** in-process only.
|
||||
*
|
||||
* Note: Node’s `JSON.stringify` runs `Buffer.toJSON()` before any replacer, so nested `Buffer`
|
||||
* values become the legacy object unless you pass a string (use `createTransferAssetStreamChunk`).
|
||||
*/
|
||||
export function decodeTransferAssetStreamData(data: unknown, encoding?: 'base64'): Buffer {
|
||||
if (encoding === 'base64' && typeof data === 'string') {
|
||||
return Buffer.from(data, 'base64');
|
||||
}
|
||||
// `encoding: 'base64'` with a non-string payload (or no encoding) uses the same fallbacks as
|
||||
// legacy peers — avoids throwing when flags and payload disagree.
|
||||
|
||||
if (Buffer.isBuffer(data)) {
|
||||
return Buffer.from(data);
|
||||
}
|
||||
|
||||
const legacyBufferData = getLegacyBufferJsonData(data);
|
||||
if (legacyBufferData) {
|
||||
return Buffer.from(legacyBufferData);
|
||||
}
|
||||
|
||||
// Wire base64 string (pull generator and any other path that stringifies a string payload).
|
||||
if (typeof data === 'string') {
|
||||
return Buffer.from(data, 'base64');
|
||||
}
|
||||
|
||||
throw new TypeError('Invalid transfer asset stream chunk payload');
|
||||
}
|
||||
|
||||
/** Approximate decoded byte size for batching (pull asset generator). */
|
||||
export function transferAssetStreamChunkByteLength(chunk: {
|
||||
action: string;
|
||||
data?: unknown;
|
||||
encoding?: 'base64';
|
||||
}): number {
|
||||
if (chunk.action !== 'stream') {
|
||||
return 0;
|
||||
}
|
||||
if (typeof chunk.data === 'string') {
|
||||
return Math.floor((chunk.data.length * 3) / 4);
|
||||
}
|
||||
if (Buffer.isBuffer(chunk.data)) {
|
||||
return chunk.data.byteLength;
|
||||
}
|
||||
|
||||
const legacyBufferData = getLegacyBufferJsonData(chunk.data);
|
||||
if (legacyBufferData) {
|
||||
if (Array.isArray(legacyBufferData)) {
|
||||
return legacyBufferData.length;
|
||||
}
|
||||
if (ArrayBuffer.isView(legacyBufferData)) {
|
||||
return legacyBufferData.byteLength;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Shared `JSON.stringify` replacer for data-transfer WebSocket frames (push and pull).
|
||||
*
|
||||
* Default `JSON.stringify` uses `Buffer.toJSON()` → `{ type: 'Buffer', data: [n,n,...] }`, which
|
||||
* allocates a large array on the peer during `JSON.parse`. Encode binary values as compact base64 strings instead.
|
||||
*
|
||||
* Note: Node runs `Buffer.prototype.toJSON` before the replacer sees a `Buffer` property, so the
|
||||
* replacer receives `{ type: 'Buffer', data: [...] }` unless the value is already a string (see
|
||||
* `createTransferAssetStreamChunk` in `transfer-asset-chunk.ts`).
|
||||
*/
|
||||
export const replacerForTransferWebSocket = (_key: string, value: unknown): unknown => {
|
||||
/** `JSON.stringify` throws on bigint; upload metadata or ORM fields may surface as BigInt. */
|
||||
if (typeof value === 'bigint') {
|
||||
return value.toString();
|
||||
}
|
||||
if (Buffer.isBuffer(value)) {
|
||||
return value.toString('base64');
|
||||
}
|
||||
if (value instanceof Uint8Array) {
|
||||
const { buffer, byteOffset, byteLength } = value;
|
||||
if (buffer == null) {
|
||||
throw new TypeError(
|
||||
'Invalid Uint8Array in transfer payload (missing underlying ArrayBuffer); cannot encode for WebSocket'
|
||||
);
|
||||
}
|
||||
return Buffer.from(buffer, byteOffset, byteLength).toString('base64');
|
||||
}
|
||||
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
||||
const v = value as NodeJS.TypedArray;
|
||||
const { buffer, byteOffset, byteLength } = v;
|
||||
if (buffer == null) {
|
||||
throw new TypeError(
|
||||
'Invalid typed array in transfer payload (missing underlying ArrayBuffer); cannot encode for WebSocket'
|
||||
);
|
||||
}
|
||||
return Buffer.from(buffer, byteOffset, byteLength).toString('base64');
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
/**
|
||||
* `JSON.stringify` invokes an own enumerable `toJSON` on the root value before replacers run. If that
|
||||
* method returns `undefined`, the whole `JSON.stringify` result is `undefined`, and `ws.send(undefined)`
|
||||
* throws ("The first argument must be of type string or an instance of Buffer... Received undefined").
|
||||
* Spreading transfer messages (`{ ...message, uuid }`) can copy an enumerable `toJSON` from user / ORM
|
||||
* objects onto the wire payload — strip it on the root object we control.
|
||||
*/
|
||||
export function stripRootToJSONMethod(payload: Record<string, unknown>): void {
|
||||
if (typeof payload.toJSON === 'function') {
|
||||
delete payload.toJSON;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize a transfer WebSocket envelope. Never returns `undefined` (unlike raw `JSON.stringify`).
|
||||
*/
|
||||
export function stringifyTransferWebSocketPayload(payload: Record<string, unknown>): string {
|
||||
stripRootToJSONMethod(payload);
|
||||
let s: string | undefined;
|
||||
try {
|
||||
s = JSON.stringify(payload, replacerForTransferWebSocket);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
throw new TypeError(`Transfer WebSocket payload could not be serialized to JSON: ${message}`);
|
||||
}
|
||||
if (typeof s !== 'string') {
|
||||
throw new TypeError(
|
||||
'Transfer WebSocket payload could not be serialized to JSON (result was undefined). Check for Symbol or other non-JSON values on the root payload.'
|
||||
);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { once } from 'node:events';
|
||||
import { finished } from 'node:stream/promises';
|
||||
import type { Writable } from 'stream';
|
||||
|
||||
/**
|
||||
* Async helper for application code that `await`s sequential writes to a `Writable`.
|
||||
*
|
||||
* 1. Waits until `writable.write` invokes its callback (chunk accepted / `_write` finished).
|
||||
* 2. If `write()` returned `false` **and** `writable.writableNeedDrain` is still true after the
|
||||
* callback, waits for `'drain'`.
|
||||
*
|
||||
* We check both: the return value tells us backpressure was signaled; `writableNeedDrain` avoids
|
||||
* awaiting `'drain'` when it already fired before we subscribed (would otherwise hang forever).
|
||||
*
|
||||
* While waiting for `'drain'`, we also race {@link finished} so destroying the writable (e.g. abort)
|
||||
* cannot leave this promise pending forever.
|
||||
*/
|
||||
export async function write(writable: Writable, chunk: unknown): Promise<void> {
|
||||
let flushed = true;
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const finish = (fn: () => void) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
writable.off('error', onError);
|
||||
fn();
|
||||
};
|
||||
const onError = (err: Error) => {
|
||||
finish(() => reject(err));
|
||||
};
|
||||
writable.once('error', onError);
|
||||
flushed = writable.write(chunk, (err) => {
|
||||
if (err) {
|
||||
// Do not reject or remove `error` here: Node may emit `error` after this callback, and
|
||||
// clearing the listener first would leave that emission unhandled.
|
||||
setImmediate(() => {
|
||||
if (!settled) {
|
||||
finish(() => reject(err));
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
finish(() => resolve());
|
||||
});
|
||||
});
|
||||
|
||||
if (!flushed && writable.writableNeedDrain) {
|
||||
// Without `finished`, awaiting only `drain` can hang forever if the writable is destroyed first.
|
||||
await Promise.race([
|
||||
once(writable, 'drain'),
|
||||
finished(writable, { readable: false, writable: true }),
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import type {
|
||||
IProviderTransferResults,
|
||||
ISourceProviderTransferResults,
|
||||
MaybePromise,
|
||||
StageTotalsEstimate,
|
||||
TransferStage,
|
||||
} from './utils';
|
||||
import type { IMetadata } from './common-entities';
|
||||
import type { IDiagnosticReporter } from '../src/utils/diagnostic';
|
||||
@@ -31,6 +33,12 @@ export interface IProvider {
|
||||
export interface ISourceProvider extends IProvider {
|
||||
results?: ISourceProviderTransferResults;
|
||||
|
||||
/**
|
||||
* Optional totals for a stage. Called by the engine after the stage read stream is created and before `stage::start`.
|
||||
* Used for CLI progress (e.g. bytes/count remaining, ETA). Omit or return null when unknown (older remotes, file providers).
|
||||
*/
|
||||
getStageTotals?(stage: TransferStage): MaybePromise<StageTotalsEstimate | null | undefined>;
|
||||
|
||||
createEntitiesReadStream?(): MaybePromise<Readable>;
|
||||
createLinksReadStream?(): MaybePromise<Readable>;
|
||||
createAssetsReadStream?(): MaybePromise<Readable>;
|
||||
|
||||
@@ -14,8 +14,9 @@ export type GetCommandParams<T extends Command> = {
|
||||
export type InitCommand = CreateCommand<
|
||||
'init',
|
||||
{
|
||||
options: Pick<ILocalStrapiDestinationProviderOptions, 'strategy' | 'restore'>;
|
||||
options?: Pick<ILocalStrapiDestinationProviderOptions, 'strategy' | 'restore'>;
|
||||
transfer: TransferMethod;
|
||||
checksums?: boolean;
|
||||
}
|
||||
>;
|
||||
export type TransferKind = InitCommand['params']['transfer'];
|
||||
|
||||
+13
-2
@@ -6,8 +6,19 @@ export type CreateTransferMessage<T extends string, U = unknown> = {
|
||||
transferID: string;
|
||||
} & U;
|
||||
|
||||
export type TransferAssetChecksum = {
|
||||
algorithm: 'sha256';
|
||||
value: string;
|
||||
};
|
||||
|
||||
export type TransferAssetFlow = { assetID: string } & (
|
||||
| { action: 'start'; data: Omit<IAsset, 'stream'> }
|
||||
| { action: 'stream'; data: Buffer }
|
||||
| { action: 'end' }
|
||||
/** Legacy in-process / default JSON: Buffer serializes to `{ type: 'Buffer'; data: number[] }` on the wire. */
|
||||
| { action: 'stream'; data: Buffer; encoding?: undefined }
|
||||
/**
|
||||
* Canonical wire form for asset bytes (push + pull). Built with `createTransferAssetStreamChunk`.
|
||||
* Decoders also accept legacy Buffer JSON and plain base64 strings without `encoding`.
|
||||
*/
|
||||
| { action: 'stream'; data: string; encoding?: 'base64' }
|
||||
| { action: 'end'; checksum?: TransferAssetChecksum }
|
||||
);
|
||||
|
||||
@@ -11,7 +11,7 @@ export type Message<T = unknown> = {
|
||||
|
||||
// Successful
|
||||
export type OKMessage = Message<{ ok: true }>;
|
||||
export type InitMessage = Message<{ transferID: string }>;
|
||||
export type InitMessage = Message<{ transferID: string; checksums?: boolean }>;
|
||||
export type EndMessage = OKMessage;
|
||||
export type StatusMessage = Message<
|
||||
| { active: true; kind: TransferKind; startedAt: number; elapsed: number }
|
||||
|
||||
+26
-11
@@ -63,21 +63,36 @@ export type TransferFilters = {
|
||||
/*
|
||||
* Progress
|
||||
*/
|
||||
export type TransferProgress = {
|
||||
[key in TransferStage]?: {
|
||||
count: number;
|
||||
bytes: number;
|
||||
startTime: number;
|
||||
endTime?: number;
|
||||
aggregates?: {
|
||||
[key: string]: {
|
||||
count: number;
|
||||
bytes: number;
|
||||
};
|
||||
export type StageTotalsEstimate = {
|
||||
totalBytes?: number;
|
||||
totalCount?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Per-stage running progress. For `assets`, optional totals describe the full transfer when known
|
||||
* (same basis as streamed data: one count per IAsset including formats; bytes = sum of binary file sizes).
|
||||
*/
|
||||
export type StageProgress = {
|
||||
count: number;
|
||||
bytes: number;
|
||||
startTime: number;
|
||||
endTime?: number;
|
||||
/** Present when the source can estimate full stage size before/during streaming (e.g. asset library). */
|
||||
totalBytes?: number;
|
||||
/** Present when the source can estimate how many items the stage will emit. */
|
||||
totalCount?: number;
|
||||
aggregates?: {
|
||||
[key: string]: {
|
||||
count: number;
|
||||
bytes: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
export type TransferProgress = {
|
||||
[key in TransferStage]?: StageProgress;
|
||||
};
|
||||
|
||||
export interface ITransferResults<S extends ISourceProvider, D extends IDestinationProvider> {
|
||||
source?: S['results'];
|
||||
destination?: D['results'];
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import * as mockDataTransfer from '@strapi/data-transfer';
|
||||
|
||||
import * as dataTransferUtils from '../../../utils/data-transfer';
|
||||
import transferAction from '../action';
|
||||
import { expectExit } from '../../__tests__/commands.test.utils';
|
||||
|
||||
@@ -9,14 +10,15 @@ jest.mock('../../../utils/data-transfer', () => {
|
||||
getTransferTelemetryPayload: jest.fn().mockReturnValue({}),
|
||||
loadersFactory: jest.fn().mockReturnValue({ updateLoader: jest.fn() }),
|
||||
formatDiagnostic: jest.fn(),
|
||||
createStrapiInstance() {
|
||||
return {
|
||||
telemetry: {
|
||||
send: jest.fn(),
|
||||
},
|
||||
contentTypes: {},
|
||||
};
|
||||
},
|
||||
createStrapiInstance: jest.fn(async () => ({
|
||||
config: {
|
||||
get: jest.fn(),
|
||||
},
|
||||
telemetry: {
|
||||
send: jest.fn(),
|
||||
},
|
||||
contentTypes: {},
|
||||
})),
|
||||
getDefaultExportName: jest.fn(() => 'default'),
|
||||
buildTransferTable: jest.fn(() => {
|
||||
return {
|
||||
@@ -34,13 +36,13 @@ jest.mock('../../../utils/data-transfer', () => {
|
||||
|
||||
// mock data transfer
|
||||
jest.mock('@strapi/data-transfer', () => {
|
||||
const acutal = jest.requireActual('@strapi/data-transfer');
|
||||
const actual = jest.requireActual('@strapi/data-transfer');
|
||||
return {
|
||||
...acutal,
|
||||
...actual,
|
||||
strapi: {
|
||||
...acutal.strapi,
|
||||
...actual.strapi,
|
||||
providers: {
|
||||
...acutal.strapi.providers,
|
||||
...actual.strapi.providers,
|
||||
createLocalStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testLocalSource' }),
|
||||
createLocalStrapiDestinationProvider: jest.fn().mockReturnValue({ name: 'testLocalDest' }),
|
||||
createRemoteStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testRemoteSource' }),
|
||||
@@ -50,19 +52,26 @@ jest.mock('@strapi/data-transfer', () => {
|
||||
},
|
||||
},
|
||||
engine: {
|
||||
...acutal.engine,
|
||||
...actual.engine,
|
||||
createTransferEngine() {
|
||||
const handlers: Record<string, Array<(...args: unknown[]) => void>> = {};
|
||||
const stream = {
|
||||
on(event: string, fn: (...args: unknown[]) => void) {
|
||||
(handlers[event] ||= []).push(fn);
|
||||
return stream;
|
||||
},
|
||||
};
|
||||
return {
|
||||
transfer: jest.fn(() => {
|
||||
transfer: jest.fn(async () => {
|
||||
// action.ts clears setInterval on this event; real stream emits after transfer.
|
||||
handlers['transfer::finish']?.forEach((fn) => fn());
|
||||
return {
|
||||
engine: {},
|
||||
};
|
||||
}),
|
||||
progress: {
|
||||
on: jest.fn(),
|
||||
stream: {
|
||||
on: jest.fn(),
|
||||
},
|
||||
stream,
|
||||
},
|
||||
sourceProvider: { name: 'testSource' },
|
||||
destinationProvider: { name: 'testDestination' },
|
||||
@@ -82,10 +91,16 @@ describe('Transfer', () => {
|
||||
// mock command utils
|
||||
|
||||
// console spies
|
||||
jest.spyOn(console, 'log').mockImplementation(() => {});
|
||||
jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
jest.spyOn(console, 'info').mockImplementation(() => {});
|
||||
jest.spyOn(console, 'error').mockImplementation(() => {});
|
||||
beforeAll(() => {
|
||||
jest.spyOn(console, 'log').mockImplementation(() => {});
|
||||
jest.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
jest.spyOn(console, 'info').mockImplementation(() => {});
|
||||
jest.spyOn(console, 'error').mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
const destinationUrl = new URL('http://one.localhost/admin');
|
||||
const destinationToken = 'test-token';
|
||||
@@ -95,6 +110,15 @@ describe('Transfer', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
(dataTransferUtils.createStrapiInstance as jest.Mock).mockImplementation(async () => ({
|
||||
config: {
|
||||
get: jest.fn(),
|
||||
},
|
||||
telemetry: {
|
||||
send: jest.fn(),
|
||||
},
|
||||
contentTypes: {},
|
||||
}));
|
||||
});
|
||||
|
||||
it('exits with error when no --to or --from is provided', async () => {
|
||||
@@ -172,6 +196,40 @@ describe('Transfer', () => {
|
||||
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
|
||||
).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('passes verifyChecksums to remote destination when --checksums is enabled', async () => {
|
||||
await expectExit(0, async () => {
|
||||
await transferAction({
|
||||
from: undefined,
|
||||
to: destinationUrl,
|
||||
toToken: destinationToken,
|
||||
checksums: true,
|
||||
} as any);
|
||||
});
|
||||
|
||||
expect(
|
||||
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
|
||||
).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
verifyChecksums: true,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('does not pass verifyChecksums to remote destination when checksums are disabled', async () => {
|
||||
await expectExit(0, async () => {
|
||||
await transferAction({
|
||||
from: undefined,
|
||||
to: destinationUrl,
|
||||
toToken: destinationToken,
|
||||
checksums: false,
|
||||
} as any);
|
||||
});
|
||||
|
||||
expect(
|
||||
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
|
||||
).toHaveBeenCalledWith(expect.not.objectContaining({ verifyChecksums: true }));
|
||||
});
|
||||
});
|
||||
|
||||
describe('--from', () => {
|
||||
@@ -210,6 +268,74 @@ describe('Transfer', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('passes verifyChecksums to remote source when --checksums is enabled', async () => {
|
||||
await expectExit(0, async () => {
|
||||
await transferAction({
|
||||
to: undefined,
|
||||
from: sourceUrl,
|
||||
fromToken: sourceToken,
|
||||
checksums: true,
|
||||
} as any);
|
||||
});
|
||||
|
||||
expect(
|
||||
mockDataTransfer.strapi.providers.createRemoteStrapiSourceProvider
|
||||
).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
verifyChecksums: true,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('does not pass verifyChecksums to remote source when checksums are disabled', async () => {
|
||||
await expectExit(0, async () => {
|
||||
await transferAction({
|
||||
to: undefined,
|
||||
from: sourceUrl,
|
||||
fromToken: sourceToken,
|
||||
checksums: false,
|
||||
} as any);
|
||||
});
|
||||
|
||||
expect(
|
||||
mockDataTransfer.strapi.providers.createRemoteStrapiSourceProvider
|
||||
).toHaveBeenCalledWith(
|
||||
expect.not.objectContaining({
|
||||
verifyChecksums: true,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('passes server.transfer.remote.assetIdleTimeoutMs to remote source provider as streamTimeout', async () => {
|
||||
(dataTransferUtils.createStrapiInstance as jest.Mock).mockImplementationOnce(async () => ({
|
||||
config: {
|
||||
get: (key: string) =>
|
||||
key === 'server.transfer.remote.assetIdleTimeoutMs' ? 99_000 : undefined,
|
||||
},
|
||||
telemetry: {
|
||||
send: jest.fn(),
|
||||
},
|
||||
contentTypes: {},
|
||||
}));
|
||||
|
||||
await expectExit(0, async () => {
|
||||
await transferAction({
|
||||
to: undefined,
|
||||
from: sourceUrl,
|
||||
fromToken: sourceToken,
|
||||
} as any);
|
||||
});
|
||||
|
||||
expect(
|
||||
mockDataTransfer.strapi.providers.createRemoteStrapiSourceProvider
|
||||
).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: sourceUrl,
|
||||
streamTimeout: 99_000,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('uses local Strapi destination when to is not specified', async () => {
|
||||
await expectExit(0, async () => {
|
||||
await transferAction({
|
||||
|
||||
@@ -27,6 +27,17 @@ const {
|
||||
},
|
||||
} = strapiDataTransfer;
|
||||
|
||||
const resolveRemotePullAssetIdleTimeoutMs = (value: unknown): number | undefined => {
|
||||
if (value == null || value === '') {
|
||||
return undefined;
|
||||
}
|
||||
const n = typeof value === 'number' ? value : Number(value);
|
||||
if (!Number.isFinite(n) || n <= 0) {
|
||||
return undefined;
|
||||
}
|
||||
return n;
|
||||
};
|
||||
|
||||
interface CmdOptions {
|
||||
from?: URL;
|
||||
fromToken: string;
|
||||
@@ -37,6 +48,7 @@ interface CmdOptions {
|
||||
exclude?: (keyof engineDataTransfer.TransferGroupFilter)[];
|
||||
throttle?: number;
|
||||
force?: boolean;
|
||||
checksums?: boolean;
|
||||
}
|
||||
/**
|
||||
* Transfer command.
|
||||
@@ -54,6 +66,7 @@ export default async (opts: CmdOptions) => {
|
||||
}
|
||||
|
||||
const strapi = await createStrapiInstance();
|
||||
const checksumsEnabled = opts.checksums !== false;
|
||||
let source;
|
||||
let destination;
|
||||
|
||||
@@ -69,6 +82,10 @@ export default async (opts: CmdOptions) => {
|
||||
exitWith(1, 'Missing token for remote destination');
|
||||
}
|
||||
|
||||
const assetIdleTimeoutMs = resolveRemotePullAssetIdleTimeoutMs(
|
||||
strapi.config.get('server.transfer.remote.assetIdleTimeoutMs')
|
||||
);
|
||||
|
||||
source = createRemoteStrapiSourceProvider({
|
||||
getStrapi: () => strapi,
|
||||
url: opts.from,
|
||||
@@ -76,6 +93,8 @@ export default async (opts: CmdOptions) => {
|
||||
type: 'token',
|
||||
token: opts.fromToken,
|
||||
},
|
||||
...(assetIdleTimeoutMs !== undefined ? { streamTimeout: assetIdleTimeoutMs } : {}),
|
||||
...(checksumsEnabled ? { verifyChecksums: true } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -101,6 +120,7 @@ export default async (opts: CmdOptions) => {
|
||||
},
|
||||
strategy: 'restore',
|
||||
restore: parseRestoreFromOptions(opts, strapi),
|
||||
...(checksumsEnabled ? { verifyChecksums: true } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -145,6 +165,19 @@ export default async (opts: CmdOptions) => {
|
||||
getAssetsBackupHandler(engine, { force: opts.force, action: 'transfer' })
|
||||
);
|
||||
|
||||
// Update more frequently to ensure elapsed time is accurate even if the stage is not progressing
|
||||
const activeStages = new Set<string>();
|
||||
const lastStageData: Record<string, any> = {};
|
||||
const interval = setInterval(() => {
|
||||
for (const stage of activeStages) {
|
||||
if (lastStageData[stage]) {
|
||||
// Clone the lastStageData and ensure endTime is undefined so elapsed uses Date.now()
|
||||
const dataCopy = { ...lastStageData[stage], endTime: undefined };
|
||||
updateLoader(stage as any, { [stage]: dataCopy });
|
||||
}
|
||||
}
|
||||
}, 100);
|
||||
|
||||
progress.on(`stage::start`, ({ stage, data }) => {
|
||||
updateLoader(stage, data).start();
|
||||
});
|
||||
@@ -154,6 +187,8 @@ export default async (opts: CmdOptions) => {
|
||||
});
|
||||
|
||||
progress.on('stage::progress', ({ stage, data }) => {
|
||||
lastStageData[stage] = data[stage];
|
||||
activeStages.add(stage);
|
||||
updateLoader(stage, data);
|
||||
});
|
||||
|
||||
@@ -161,6 +196,9 @@ export default async (opts: CmdOptions) => {
|
||||
updateLoader(stage, data).fail();
|
||||
});
|
||||
|
||||
progress.on('transfer::finish', () => clearInterval(interval));
|
||||
progress.on('transfer::error', () => clearInterval(interval));
|
||||
|
||||
progress.on('transfer::start', async () => {
|
||||
console.log(`Starting transfer...`);
|
||||
|
||||
|
||||
@@ -36,6 +36,12 @@ const command = () => {
|
||||
.addOption(
|
||||
new Option('--to-token <token>', `Transfer token for the remote Strapi destination`)
|
||||
)
|
||||
.addOption(
|
||||
new Option(
|
||||
'--no-checksums',
|
||||
'Disable end-to-end asset checksum verification for assets transfer'
|
||||
)
|
||||
)
|
||||
.addOption(new Option('--verbose', 'Enable verbose logs'))
|
||||
.addOption(forceOption)
|
||||
.addOption(excludeOption)
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { loadersFactory } from '../data-transfer';
|
||||
|
||||
jest.mock('ora', () =>
|
||||
jest.fn(() => ({
|
||||
start: jest.fn(),
|
||||
succeed: jest.fn(),
|
||||
fail: jest.fn(),
|
||||
text: '',
|
||||
}))
|
||||
);
|
||||
|
||||
describe('loadersFactory / updateLoader (transfer progress line)', () => {
|
||||
test('includes count and size totals when totalBytes and totalCount are set', () => {
|
||||
const { updateLoader, getLoader } = loadersFactory();
|
||||
const t0 = 1_700_000_000_000;
|
||||
const dateSpy = jest.spyOn(Date, 'now').mockReturnValue(t0 + 1000);
|
||||
|
||||
updateLoader('assets', {
|
||||
assets: {
|
||||
startTime: t0,
|
||||
bytes: 100,
|
||||
count: 1,
|
||||
totalBytes: 400,
|
||||
totalCount: 4,
|
||||
},
|
||||
});
|
||||
|
||||
const text = getLoader('assets').text;
|
||||
expect(text).toContain('1 / 4');
|
||||
expect(text).toMatch(/size:.*\/.*400/);
|
||||
|
||||
dateSpy.mockRestore();
|
||||
});
|
||||
|
||||
test('includes eta when elapsed >= 500ms, bytes in range, and totalBytes known', () => {
|
||||
const { updateLoader, getLoader } = loadersFactory();
|
||||
const t0 = 1_700_000_000_000;
|
||||
const dateSpy = jest.spyOn(Date, 'now').mockReturnValue(t0 + 1000);
|
||||
|
||||
updateLoader('assets', {
|
||||
assets: {
|
||||
startTime: t0,
|
||||
bytes: 500,
|
||||
count: 1,
|
||||
totalBytes: 1500,
|
||||
totalCount: 2,
|
||||
},
|
||||
});
|
||||
|
||||
expect(getLoader('assets').text).toContain('eta ~');
|
||||
|
||||
dateSpy.mockRestore();
|
||||
});
|
||||
|
||||
test('omits eta when stage has finished (endTime set)', () => {
|
||||
const { updateLoader, getLoader } = loadersFactory();
|
||||
const t0 = 1_700_000_000_000;
|
||||
const dateSpy = jest.spyOn(Date, 'now').mockReturnValue(t0 + 2000);
|
||||
|
||||
updateLoader('assets', {
|
||||
assets: {
|
||||
startTime: t0,
|
||||
endTime: t0 + 2000,
|
||||
bytes: 1500,
|
||||
count: 2,
|
||||
totalBytes: 1500,
|
||||
totalCount: 2,
|
||||
},
|
||||
});
|
||||
|
||||
expect(getLoader('assets').text).not.toContain('eta ~');
|
||||
|
||||
dateSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -46,8 +46,8 @@ describe('logger', () => {
|
||||
|
||||
describe('formatDiagnostic', () => {
|
||||
it('only creates a single logger', () => {
|
||||
const info = true; // so we info reports are called
|
||||
const diagnosticReporter = formatDiagnostic('export', info);
|
||||
const verbose = true;
|
||||
const diagnosticReporter = formatDiagnostic('export', verbose);
|
||||
|
||||
// Use the diagnostic reporter to log different levels of messages
|
||||
diagnosticReporter({
|
||||
@@ -83,29 +83,12 @@ describe('logger', () => {
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(expect.stringContaining('Test error'));
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1);
|
||||
expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('Test warning'));
|
||||
expect(mockLogger.info).toHaveBeenCalledTimes(1);
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(expect.stringContaining('Test info'));
|
||||
const infoMessages = mockLogger.info.mock.calls.map((c) => String(c[0]));
|
||||
expect(infoMessages.some((m) => m.includes('Diagnostic log file'))).toBe(true);
|
||||
expect(infoMessages.some((m) => m.includes('Test info'))).toBe(true);
|
||||
});
|
||||
|
||||
it('does not log info if info is false', () => {
|
||||
const diagnosticReporter = formatDiagnostic('export');
|
||||
|
||||
diagnosticReporter({
|
||||
kind: 'info',
|
||||
details: {
|
||||
message: 'Test info',
|
||||
createdAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
// Verify createLogger is not called
|
||||
expect(createLogger).not.toBeCalled();
|
||||
|
||||
// Verify the mock logger does not receive logs
|
||||
expect(mockLogger.info).not.toBeCalled();
|
||||
});
|
||||
|
||||
it('does not log info if info is false, but logs error and warning logs', () => {
|
||||
it('writes error, warning, and info to the diagnostic file when verbose (console) is false', () => {
|
||||
const diagnosticReporter = formatDiagnostic('export');
|
||||
|
||||
// Use the diagnostic reporter to log different levels of messages
|
||||
@@ -143,8 +126,8 @@ describe('logger', () => {
|
||||
expect(mockLogger.warn).toHaveBeenCalledTimes(1);
|
||||
expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('Test warning'));
|
||||
|
||||
// Verify the mock info logger does not receive logs
|
||||
expect(mockLogger.info).not.toBeCalled();
|
||||
const infoMessages = mockLogger.info.mock.calls.map((c) => String(c[0]));
|
||||
expect(infoMessages.some((m) => m.includes('Test info'))).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import chalk from 'chalk';
|
||||
import path from 'node:path';
|
||||
import Table from 'cli-table3';
|
||||
import { Command, Option } from 'commander';
|
||||
import { configs, createLogger, type winston, formats } from '@strapi/logger';
|
||||
@@ -8,7 +9,7 @@ import { merge } from 'lodash/fp';
|
||||
import type { Core } from '@strapi/types';
|
||||
import { engine as engineDataTransfer, strapi as strapiDataTransfer } from '@strapi/data-transfer';
|
||||
|
||||
import { readableBytes, exitWith } from './helpers';
|
||||
import { readableBytes, readableTime, exitWith } from './helpers';
|
||||
import { getParseListWithChoices, parseInteger, confirmMessage } from './commander';
|
||||
|
||||
const {
|
||||
@@ -209,24 +210,36 @@ const errorColors = {
|
||||
|
||||
const formatDiagnostic = (
|
||||
operation: string,
|
||||
info?: boolean
|
||||
verbose?: boolean
|
||||
): Parameters<engineDataTransfer.TransferEngine['diagnostics']['onDiagnostic']>[0] => {
|
||||
// Create log file for all incoming diagnostics
|
||||
let logger: undefined | winston.Logger;
|
||||
let logFileBasename: string | undefined;
|
||||
|
||||
const getLogger = () => {
|
||||
if (!logger) {
|
||||
logFileBasename = `${operation}_${Date.now()}.log`;
|
||||
const absoluteLogPath = path.resolve(process.cwd(), logFileBasename);
|
||||
|
||||
logger = createLogger(
|
||||
configs.createOutputFileConfiguration(`${operation}_${Date.now()}.log`, {
|
||||
level: 'info',
|
||||
format: formats?.detailedLogs,
|
||||
})
|
||||
configs.createOutputFileConfiguration(
|
||||
logFileBasename,
|
||||
{
|
||||
level: 'info',
|
||||
format: formats?.detailedLogs,
|
||||
},
|
||||
{
|
||||
consoleLevel: verbose ? 'info' : 'warn',
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
logger.info(
|
||||
`[${operation}] Diagnostic log file: ${absoluteLogPath} (info-level messages are written here even without --verbose)`
|
||||
);
|
||||
}
|
||||
return logger;
|
||||
};
|
||||
|
||||
// We don't want to write a log file until there is something to be logged
|
||||
|
||||
return ({ details, kind }) => {
|
||||
try {
|
||||
if (kind === 'error') {
|
||||
@@ -237,7 +250,7 @@ const formatDiagnostic = (
|
||||
|
||||
getLogger().error(errorMessage);
|
||||
}
|
||||
if (kind === 'info' && info) {
|
||||
if (kind === 'info') {
|
||||
const { message, params, origin } = details;
|
||||
|
||||
const msg = `[${origin ?? 'transfer'}] ${message}\n${params ? JSON.stringify(params, null, 2) : ''}`;
|
||||
@@ -265,6 +278,8 @@ type Data = {
|
||||
endTime?: number;
|
||||
bytes?: number;
|
||||
count?: number;
|
||||
totalBytes?: number;
|
||||
totalCount?: number;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -279,14 +294,40 @@ const loadersFactory = (defaultLoaders: Loaders = {} as Loaders) => {
|
||||
const elapsedTime = stageData?.startTime
|
||||
? (stageData?.endTime || Date.now()) - stageData.startTime
|
||||
: 0;
|
||||
const size = `size: ${readableBytes(stageData?.bytes ?? 0)}`;
|
||||
const elapsed = `elapsed: ${elapsedTime} ms`;
|
||||
const speed =
|
||||
elapsedTime > 0 ? `(${readableBytes(((stageData?.bytes ?? 0) * 1000) / elapsedTime)}/s)` : '';
|
||||
const bytes = stageData?.bytes ?? 0;
|
||||
const count = stageData?.count ?? 0;
|
||||
const totalBytes = stageData?.totalBytes;
|
||||
const totalCount = stageData?.totalCount;
|
||||
|
||||
loaders[stage].text = `${stage}: ${stageData?.count ?? 0} transferred (${size}) (${elapsed}) ${
|
||||
const countLabel =
|
||||
totalCount != null && totalCount > 0 ? `${count} / ${totalCount}` : String(count);
|
||||
const size =
|
||||
totalBytes != null && totalBytes > 0
|
||||
? `size: ${readableBytes(bytes)} / ${readableBytes(totalBytes)}`
|
||||
: `size: ${readableBytes(bytes)}`;
|
||||
const elapsed = `elapsed: ${readableTime(elapsedTime ?? 0)}`;
|
||||
const speed = elapsedTime > 0 ? `(${readableBytes((bytes * 1000) / elapsedTime)}/s)` : '';
|
||||
|
||||
let eta = '';
|
||||
if (
|
||||
!stageData?.endTime &&
|
||||
totalBytes != null &&
|
||||
totalBytes > 0 &&
|
||||
bytes < totalBytes &&
|
||||
elapsedTime >= 500 &&
|
||||
bytes > 0
|
||||
) {
|
||||
const speedBps = bytes / elapsedTime;
|
||||
const remaining = totalBytes - bytes;
|
||||
const etaMs = remaining / speedBps;
|
||||
if (Number.isFinite(etaMs) && etaMs > 0 && etaMs < 86400000) {
|
||||
eta = ` eta ~${readableTime(etaMs)}`;
|
||||
}
|
||||
}
|
||||
|
||||
loaders[stage].text = `${stage}: ${countLabel} transferred (${size}) (${elapsed}) ${
|
||||
!stageData?.endTime ? speed : ''
|
||||
}`;
|
||||
}${eta}`;
|
||||
|
||||
return loaders[stage];
|
||||
};
|
||||
|
||||
@@ -19,13 +19,40 @@ const readableBytes = (bytes: number, decimals = 1, padStart = 0) => {
|
||||
return '0';
|
||||
}
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(bytesPerKb));
|
||||
const result = `${parseFloat((bytes / bytesPerKb ** i).toFixed(decimals))} ${sizes[i].padStart(
|
||||
2
|
||||
)}`;
|
||||
const result = `${(bytes / bytesPerKb ** i).toFixed(decimals)} ${sizes[i].padStart(2)}`;
|
||||
|
||||
return result.padStart(padStart);
|
||||
};
|
||||
|
||||
// Helper to floor a number to a given number of decimal places
|
||||
function floorToDecimal(value: number, decimals: number) {
|
||||
const factor = 10 ** decimals;
|
||||
return Math.floor(value * factor) / factor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert elapsed time to a human readable formatted string, for example "1024" becomes "1s"
|
||||
*/
|
||||
const readableTime = (elapsedTime: number, decimals = 1, padStart = 0): string => {
|
||||
let value: number;
|
||||
let unit: string;
|
||||
|
||||
if (elapsedTime >= 60000) {
|
||||
value = elapsedTime / 60000;
|
||||
unit = 'm';
|
||||
} else if (elapsedTime >= 1000) {
|
||||
value = elapsedTime / 1000;
|
||||
unit = 's';
|
||||
} else {
|
||||
value = elapsedTime;
|
||||
unit = 'ms';
|
||||
}
|
||||
|
||||
const floored = floorToDecimal(value, decimals);
|
||||
const result = `${floored.toFixed(decimals)}${unit}`;
|
||||
return result.padStart(padStart);
|
||||
};
|
||||
|
||||
interface ExitWithOptions {
|
||||
logger?: Console;
|
||||
prc?: NodeJS.Process;
|
||||
@@ -196,6 +223,7 @@ export {
|
||||
assertUrlHasProtocol,
|
||||
ifOptions,
|
||||
readableBytes,
|
||||
readableTime,
|
||||
runAction,
|
||||
assertCwdContainsStrapiProject,
|
||||
notifyExperimentalCommand,
|
||||
|
||||
@@ -28,6 +28,12 @@ export interface ServerTransfer {
|
||||
remote?:
|
||||
| {
|
||||
enabled?: boolean;
|
||||
/**
|
||||
* Max milliseconds without forward progress while pulling assets from a remote instance
|
||||
* (`strapi transfer --from …`). Maps to the remote source provider stall timeout.
|
||||
* Omit to use the package default (typically several minutes for large files over JSON/WS).
|
||||
*/
|
||||
assetIdleTimeoutMs?: number;
|
||||
}
|
||||
| undefined;
|
||||
}
|
||||
|
||||
@@ -3,16 +3,26 @@ import { transports, LoggerOptions } from 'winston';
|
||||
import { LEVEL_LABEL, LEVELS } from '../constants';
|
||||
import { prettyPrint, excludeColors } from '../formats';
|
||||
|
||||
export type OutputFileLoggerOptions = {
|
||||
/** Console only: omit noisy `info` when e.g. `warn` (file transport still uses `fileTransportOptions.level`). */
|
||||
consoleLevel?: string;
|
||||
};
|
||||
|
||||
export default (
|
||||
filename: string,
|
||||
fileTransportOptions: transports.FileTransportOptions = {}
|
||||
fileTransportOptions: transports.FileTransportOptions = {},
|
||||
options?: OutputFileLoggerOptions
|
||||
): LoggerOptions => {
|
||||
const consoleLevel = options?.consoleLevel ?? LEVEL_LABEL;
|
||||
|
||||
return {
|
||||
level: LEVEL_LABEL,
|
||||
levels: LEVELS,
|
||||
format: prettyPrint(),
|
||||
transports: [
|
||||
new transports.Console(),
|
||||
new transports.Console({
|
||||
level: consoleLevel,
|
||||
}),
|
||||
new transports.File({
|
||||
level: 'error',
|
||||
filename,
|
||||
|
||||
@@ -18,7 +18,7 @@ const createTestTransferToken = async (strapi) => {
|
||||
name: 'TestToken',
|
||||
description: 'Transfer token used to seed the e2e database',
|
||||
lifespan: null,
|
||||
permissions: ['push'],
|
||||
permissions: ['push', 'pull'],
|
||||
accessKey: CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
|
||||
});
|
||||
}
|
||||
|
||||
+79
-1
@@ -19,6 +19,84 @@ The `-c X` option can be used to limit the number of concurrently running domain
|
||||
|
||||
If any changes are made to the template, or other issues are being encountered, try removing and regenerating the test apps by using `yarn test:cli:clean` before running the tests.
|
||||
|
||||
### Passing Jest or Playwright CLI flags (`--testPathPattern`, etc.)
|
||||
|
||||
`tests/scripts/run-tests.js` parses **runner-only** options (`-d` / `--domains`, `-c` / `--concurrency`, `-f` / `--setup`) with yargs, then spawns **Jest** (CLI tests) or **Playwright** (e2e). Any other flags (for example `--testPathPattern`, `--grep`) are forwarded to that underlying tool.
|
||||
|
||||
- With **Yarn**: `yarn test:cli -d strapi --testPathPattern=pull-remote` (or `push-remote`) works.
|
||||
- With **npm**, you must pass arguments through npm:
|
||||
`npm run test:cli -- -d strapi --testPathPattern=pull-remote`
|
||||
- You can still use `--` so everything after it is forwarded as raw argv, e.g.
|
||||
`yarn test:cli -d strapi -- --testPathPattern pull-remote`
|
||||
|
||||
### Environment variables
|
||||
|
||||
Jest is started from `tests/utils/runners/cli-runner.js` via **execa** with a **small** `env` object: only what the runner must inject (`TEST_APPS`, `JWT_SECRET`). Execa’s default **`extendEnv: true`** merges that object **on top of** the parent `process.env`.
|
||||
|
||||
So you do **not** need to whitelist variables in the runner. For **stress runs**, set `TRANSFER_CLI_MEDIA_COUNT` / `TRANSFER_CLI_MEDIA_BYTES` in the same shell as `yarn test:cli` so the seed scripts see them (defaults are tiny: 2 files × 2048 bytes).
|
||||
|
||||
The same merge behavior applies to **e2e**: `tests/utils/runners/browser-runner.js` only sets `PORT`, `HOST`, `TEST_APP_PATH`, and `STRAPI_DISABLE_EE`; the rest of the environment is inherited from how you invoked `yarn test:e2e`.
|
||||
|
||||
### Remote transfer e2e (pull and push, generated media)
|
||||
|
||||
The `strapi` domain reserves **two** test apps (`tests/cli/tests/strapi/config.js`) for:
|
||||
|
||||
| Suite | File | Direction |
|
||||
| ----- | --------------------------------------- | ------------------------- |
|
||||
| Pull | `data-transfer/pull-remote.test.cli.js` | Remote → local (`--from`) |
|
||||
| Push | `data-transfer/push-remote.test.cli.js` | Local → remote (`--to`) |
|
||||
|
||||
Both run on every `yarn test:cli -d strapi` with **small** synthetic media (defaults: `TRANSFER_CLI_MEDIA_COUNT=2`, `TRANSFER_CLI_MEDIA_BYTES=2048`). No env flag is required to run them.
|
||||
|
||||
Assertions go beyond row counts: each suite compares **Strapi’s stored content `hash` plus `size`** for every seeded upload row between source and destination (see `tests/utils/cli-transfer-remote-e2e/upload-db.js`), so corrupted or truncated assets should fail even if the file count stayed right.
|
||||
|
||||
**Stress testing only** — raise total payload by exporting larger values for the same variables before invoking the CLI tests:
|
||||
|
||||
- `TRANSFER_CLI_MEDIA_COUNT` — number of synthetic files (default `2`)
|
||||
- `TRANSFER_CLI_MEDIA_BYTES` — bytes per file (default `2048`)
|
||||
- `CLI_TRANSFER_REMOTE_PORT` — remote Strapi port (default `13710`; legacy alias `CLI_TRANSFER_PULL_REMOTE_PORT`)
|
||||
|
||||
Other optional overrides: `CLI_TRANSFER_REMOTE_JEST_TIMEOUT_MS`, `CLI_TRANSFER_REMOTE_RUNNER_TIMEOUT_MS` (legacy `CLI_TRANSFER_PULL_*`).
|
||||
|
||||
This flow is **Jest / `yarn test:cli`**, not Playwright. The IDE Playwright runner does not execute these suites.
|
||||
|
||||
**Pull** (seed on **remote**), default small run:
|
||||
|
||||
```bash
|
||||
yarn test:cli -d strapi --testPathPattern=pull-remote
|
||||
```
|
||||
|
||||
**Push** (seed on **local**), default small run:
|
||||
|
||||
```bash
|
||||
yarn test:cli -d strapi --testPathPattern=push-remote
|
||||
```
|
||||
|
||||
Stress — larger library (10 × 512 KiB), **pull**:
|
||||
|
||||
```bash
|
||||
TRANSFER_CLI_MEDIA_COUNT=10 TRANSFER_CLI_MEDIA_BYTES=524288 \
|
||||
yarn test:cli -d strapi --testPathPattern=pull-remote
|
||||
```
|
||||
|
||||
Same sizes, **push**:
|
||||
|
||||
```bash
|
||||
TRANSFER_CLI_MEDIA_COUNT=10 TRANSFER_CLI_MEDIA_BYTES=524288 \
|
||||
yarn test:cli -d strapi --testPathPattern=push-remote
|
||||
```
|
||||
|
||||
Stress — ~2 GiB total (20 × 100 MiB; `104857600` = 100 × 1024² bytes):
|
||||
|
||||
```bash
|
||||
TRANSFER_CLI_MEDIA_COUNT=20 TRANSFER_CLI_MEDIA_BYTES=104857600 \
|
||||
yarn test:cli -d strapi --testPathPattern=pull-remote
|
||||
```
|
||||
|
||||
Large seeds need **plenty of free disk** under `test-apps/cli`. The **strapi** CLI domain uses a **30 minute** outer Jest budget by default (pull + push are heavy); it grows automatically when `TRANSFER_CLI_MEDIA_BYTES × TRANSFER_CLI_MEDIA_COUNT` exceeds 10 MiB or 100 MiB (up to **4 hours**). Other CLI domains keep a **2 minute** outer budget.
|
||||
|
||||
Implementation for these tests lives in **`tests/utils/cli-transfer-remote-e2e/`** (single package: seeding, SQLite checks, HTTP wait, timeouts). The **`tests/utils/seed-cli-transfer-media.js`** file is only a thin CLI entry that forwards to that package.
|
||||
|
||||
## Writing tests
|
||||
|
||||
The [coffee](https://github.com/node-modules/coffee) library is used to run commands and expect input, complete prompts, etc. Please see their documentation for more details.
|
||||
@@ -37,7 +115,7 @@ Your CLI commands being tested can then be run in that directory.
|
||||
|
||||
As the CLI generally does not require a running Strapi app, this is not managed by the CLI testing tool.
|
||||
|
||||
After tests for remote data-transfer are implemented, there will be utility functions available to assist in running one or more of the test apps in the background while other tests are run against it.
|
||||
For remote pull/push transfer, the suites under `data-transfer/pull-remote.test.cli.js` and `push-remote.test.cli.js` start a Strapi process in the background on a fixed port; see **Remote transfer e2e** above.
|
||||
|
||||
### Structure
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
module.exports = () => {
|
||||
return {
|
||||
testApps: 1,
|
||||
// Second app is reserved for remote transfer CLI e2e (pull-remote / push-remote).
|
||||
testApps: 2,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* End-to-end CLI test: pull data (including upload files) from a running remote Strapi into a
|
||||
* second local test app. Runs on every strapi CLI pass with tiny seeded media by default.
|
||||
*
|
||||
* Stress only — increase synthetic file count/size:
|
||||
* TRANSFER_CLI_MEDIA_COUNT (default 2)
|
||||
* TRANSFER_CLI_MEDIA_BYTES (default 2048)
|
||||
*
|
||||
* Helpers: tests/utils/cli-transfer-remote-e2e/
|
||||
*/
|
||||
|
||||
const { spawn } = require('child_process');
|
||||
const coffee = require('coffee');
|
||||
const execa = require('execa');
|
||||
|
||||
const utils = require('../../../../utils');
|
||||
const {
|
||||
APP_TEMPLATE_CONSTANTS,
|
||||
getRemotePort,
|
||||
countUploadFiles,
|
||||
getSeedUploadSignature,
|
||||
jestSuiteTimeoutMs,
|
||||
seedTransferTestMedia,
|
||||
waitForHttpOk,
|
||||
} = require('../../../../utils/cli-transfer-remote-e2e');
|
||||
// eslint-disable-next-line import/extensions
|
||||
const { resetDatabaseAndImportDataFromPathProgrammatic } = require('../../../../utils/dts-import');
|
||||
|
||||
const REMOTE_PORT = getRemotePort();
|
||||
|
||||
describe('strapi transfer pull — remote to local (generated media)', () => {
|
||||
jest.setTimeout(jestSuiteTimeoutMs());
|
||||
|
||||
let remotePath;
|
||||
let localPath;
|
||||
let remoteChild;
|
||||
const { CUSTOM_TRANSFER_TOKEN_ACCESS_KEY } = require(APP_TEMPLATE_CONSTANTS);
|
||||
|
||||
beforeAll(async () => {
|
||||
const apps = utils.instances.getTestApps();
|
||||
remotePath = apps[0];
|
||||
localPath = apps[1];
|
||||
if (!remotePath || !localPath) {
|
||||
throw new Error('Expected TEST_APPS to list two app paths (strapi domain testApps: 2)');
|
||||
}
|
||||
|
||||
await resetDatabaseAndImportDataFromPathProgrammatic(remotePath, 'with-admin');
|
||||
await resetDatabaseAndImportDataFromPathProgrammatic(localPath, 'with-admin');
|
||||
|
||||
await seedTransferTestMedia(remotePath);
|
||||
|
||||
await execa('npm', ['run', '-s', 'build'], {
|
||||
cwd: remotePath,
|
||||
stdio: 'inherit',
|
||||
env: { ...process.env, PATH: process.env.PATH },
|
||||
});
|
||||
|
||||
const remoteFilesAfterSeed = countUploadFiles(remotePath);
|
||||
expect(remoteFilesAfterSeed).toBeGreaterThan(0);
|
||||
|
||||
remoteChild = spawn('npm', ['run', '-s', 'start'], {
|
||||
cwd: remotePath,
|
||||
env: {
|
||||
...process.env,
|
||||
PORT: REMOTE_PORT,
|
||||
HOST: '127.0.0.1',
|
||||
},
|
||||
stdio: 'ignore',
|
||||
});
|
||||
|
||||
const base = `http://127.0.0.1:${REMOTE_PORT}`;
|
||||
await waitForHttpOk(`${base}/admin`);
|
||||
|
||||
const resetTok = await fetch(`${base}/api/config/resettransfertoken`, { method: 'POST' });
|
||||
if (!resetTok.ok) {
|
||||
throw new Error(`resettransfertoken failed: ${resetTok.status}`);
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (remoteChild && !remoteChild.killed) {
|
||||
remoteChild.kill('SIGTERM');
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
if (!remoteChild.killed) {
|
||||
remoteChild.kill('SIGKILL');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('pulls upload files from remote (counts + Strapi content hashes match)', async () => {
|
||||
const remoteSig = getSeedUploadSignature(remotePath);
|
||||
expect(remoteSig.files.length).toBeGreaterThan(0);
|
||||
expect(getSeedUploadSignature(localPath).files).toHaveLength(0);
|
||||
|
||||
const remoteFiles = countUploadFiles(remotePath);
|
||||
const fromUrl = `http://127.0.0.1:${REMOTE_PORT}/admin`;
|
||||
|
||||
await coffee
|
||||
.spawn(
|
||||
'npm',
|
||||
[
|
||||
'run',
|
||||
'-s',
|
||||
'strapi',
|
||||
'--',
|
||||
'transfer',
|
||||
'--from',
|
||||
fromUrl,
|
||||
'--from-token',
|
||||
CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
|
||||
'--force',
|
||||
],
|
||||
{ cwd: localPath }
|
||||
)
|
||||
.expect('code', 0)
|
||||
.end();
|
||||
|
||||
const localFiles = countUploadFiles(localPath);
|
||||
expect(localFiles).toBe(remoteFiles);
|
||||
expect(getSeedUploadSignature(localPath).files).toEqual(remoteSig.files);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,132 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* End-to-end CLI test: push data (including upload files) from the local Strapi app to a running
|
||||
* remote. Runs on every strapi CLI pass with tiny seeded media by default.
|
||||
*
|
||||
* Stress only — increase synthetic file count/size:
|
||||
* TRANSFER_CLI_MEDIA_COUNT (default 2)
|
||||
* TRANSFER_CLI_MEDIA_BYTES (default 2048)
|
||||
*
|
||||
* Remote port: CLI_TRANSFER_REMOTE_PORT (legacy: CLI_TRANSFER_PULL_REMOTE_PORT), default 13710
|
||||
*
|
||||
* Helpers: tests/utils/cli-transfer-remote-e2e/
|
||||
*/
|
||||
|
||||
const { spawn } = require('child_process');
|
||||
const coffee = require('coffee');
|
||||
const execa = require('execa');
|
||||
|
||||
const utils = require('../../../../utils');
|
||||
const {
|
||||
APP_TEMPLATE_CONSTANTS,
|
||||
getRemotePort,
|
||||
countUploadFiles,
|
||||
getSeedUploadSignature,
|
||||
jestSuiteTimeoutMs,
|
||||
seedTransferTestMedia,
|
||||
waitForHttpOk,
|
||||
} = require('../../../../utils/cli-transfer-remote-e2e');
|
||||
// eslint-disable-next-line import/extensions
|
||||
const { resetDatabaseAndImportDataFromPathProgrammatic } = require('../../../../utils/dts-import');
|
||||
|
||||
const REMOTE_PORT = getRemotePort();
|
||||
|
||||
describe('strapi transfer push — local to remote (generated media)', () => {
|
||||
jest.setTimeout(jestSuiteTimeoutMs());
|
||||
|
||||
let remotePath;
|
||||
let localPath;
|
||||
let remoteChild;
|
||||
const { CUSTOM_TRANSFER_TOKEN_ACCESS_KEY } = require(APP_TEMPLATE_CONSTANTS);
|
||||
|
||||
beforeAll(async () => {
|
||||
const apps = utils.instances.getTestApps();
|
||||
remotePath = apps[0];
|
||||
localPath = apps[1];
|
||||
if (!remotePath || !localPath) {
|
||||
throw new Error('Expected TEST_APPS to list two app paths (strapi domain testApps: 2)');
|
||||
}
|
||||
|
||||
await resetDatabaseAndImportDataFromPathProgrammatic(remotePath, 'with-admin');
|
||||
await resetDatabaseAndImportDataFromPathProgrammatic(localPath, 'with-admin');
|
||||
|
||||
await seedTransferTestMedia(localPath);
|
||||
|
||||
await execa('npm', ['run', '-s', 'build'], {
|
||||
cwd: localPath,
|
||||
stdio: 'inherit',
|
||||
env: { ...process.env, PATH: process.env.PATH },
|
||||
});
|
||||
|
||||
await execa('npm', ['run', '-s', 'build'], {
|
||||
cwd: remotePath,
|
||||
stdio: 'inherit',
|
||||
env: { ...process.env, PATH: process.env.PATH },
|
||||
});
|
||||
|
||||
const localFilesAfterSeed = countUploadFiles(localPath);
|
||||
expect(localFilesAfterSeed).toBeGreaterThan(0);
|
||||
|
||||
remoteChild = spawn('npm', ['run', '-s', 'start'], {
|
||||
cwd: remotePath,
|
||||
env: {
|
||||
...process.env,
|
||||
PORT: REMOTE_PORT,
|
||||
HOST: '127.0.0.1',
|
||||
},
|
||||
stdio: 'ignore',
|
||||
});
|
||||
|
||||
const base = `http://127.0.0.1:${REMOTE_PORT}`;
|
||||
await waitForHttpOk(`${base}/admin`);
|
||||
|
||||
const resetTok = await fetch(`${base}/api/config/resettransfertoken`, { method: 'POST' });
|
||||
if (!resetTok.ok) {
|
||||
throw new Error(`resettransfertoken failed: ${resetTok.status}`);
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (remoteChild && !remoteChild.killed) {
|
||||
remoteChild.kill('SIGTERM');
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
if (!remoteChild.killed) {
|
||||
remoteChild.kill('SIGKILL');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('pushes upload files to remote (counts + Strapi content hashes match)', async () => {
|
||||
const localSig = getSeedUploadSignature(localPath);
|
||||
expect(localSig.files.length).toBeGreaterThan(0);
|
||||
expect(getSeedUploadSignature(remotePath).files).toHaveLength(0);
|
||||
|
||||
const localFiles = countUploadFiles(localPath);
|
||||
const toUrl = `http://127.0.0.1:${REMOTE_PORT}/admin`;
|
||||
|
||||
await coffee
|
||||
.spawn(
|
||||
'npm',
|
||||
[
|
||||
'run',
|
||||
'-s',
|
||||
'strapi',
|
||||
'--',
|
||||
'transfer',
|
||||
'--to',
|
||||
toUrl,
|
||||
'--to-token',
|
||||
CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
|
||||
'--force',
|
||||
],
|
||||
{ cwd: localPath }
|
||||
)
|
||||
.expect('code', 0)
|
||||
.end();
|
||||
|
||||
const remoteFiles = countUploadFiles(remotePath);
|
||||
expect(remoteFiles).toBe(localFiles);
|
||||
expect(getSeedUploadSignature(remotePath).files).toEqual(localSig.files);
|
||||
});
|
||||
});
|
||||
@@ -19,6 +19,12 @@ The `-c X` option can be used to limit the number of concurrent test apps, where
|
||||
|
||||
If any changes are made to the template, or other issues are being encountered, try removing and regenerating the test apps by using `yarn test:e2e:clean` before running the tests.
|
||||
|
||||
Options meant for Playwright (for example `--grep`) are forwarded by the unified runner after the runner-only flags (`-d`, `-c`, `-f`). With **npm**, pass them after `--`: `npm run test:e2e -- -d settings --grep "My test"`. See `tests/cli/README.md` for the same pattern on CLI/Jest.
|
||||
|
||||
### Environment variables
|
||||
|
||||
Playwright is spawned from `tests/utils/runners/browser-runner.js` with a **small** `env` override (`PORT`, `HOST`, `TEST_APP_PATH`, `STRAPI_DISABLE_EE`). **execa** merges that with the parent environment by default, so other variables you export before `yarn test:e2e` still reach Playwright and the app under test. Details and the parallel Jest behaviour are in `tests/cli/README.md` → **Environment variables**.
|
||||
|
||||
## Additional Documentation
|
||||
|
||||
See contributor docs in `docs/docs/guides/e2e` for more detailed information about writing and maintaining e2e tests.
|
||||
|
||||
@@ -16,6 +16,7 @@ const { runPlaywright } = require('../utils/runners/browser-runner');
|
||||
const {
|
||||
loadDomainConfigs,
|
||||
calculateTestAppsRequired,
|
||||
buildForwardedRunnerArgs,
|
||||
runCLI,
|
||||
} = require('../utils/runners/cli-runner');
|
||||
const { createConfig } = require('../../playwright.base.config');
|
||||
@@ -109,6 +110,8 @@ yargs
|
||||
|
||||
const { concurrency, domains: selectedDomains, setup, updateSnapshot } = testYargs;
|
||||
|
||||
const forwardedRunnerArgs = buildForwardedRunnerArgs(testYargs);
|
||||
|
||||
/**
|
||||
* Publishing all packages to the yalc store
|
||||
*/
|
||||
@@ -396,7 +399,7 @@ module.exports = config
|
||||
cwd,
|
||||
port,
|
||||
testAppPath,
|
||||
testArgs: testYargs._,
|
||||
testArgs: forwardedRunnerArgs,
|
||||
});
|
||||
})
|
||||
);
|
||||
@@ -438,7 +441,8 @@ module.exports = config
|
||||
domainDir,
|
||||
jestConfigPath,
|
||||
testApps,
|
||||
testArgs: [...(updateSnapshot ? ['-u'] : []), ...testYargs._],
|
||||
testArgs: [...(updateSnapshot ? ['-u'] : []), ...forwardedRunnerArgs],
|
||||
domain,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('Test suite failed for', domain);
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Upload filenames produced by seed-media.js; upload-db.js filters on the same prefix.
|
||||
* Keep in sync: only this module defines the string.
|
||||
*/
|
||||
const SEED_UPLOAD_NAME_PREFIX = 'cli-pull-seed-';
|
||||
/** SQLite LIKE pattern (includes wildcard). */
|
||||
const SEED_UPLOAD_NAME_LIKE = `${SEED_UPLOAD_NAME_PREFIX}%`;
|
||||
|
||||
module.exports = {
|
||||
SEED_UPLOAD_NAME_PREFIX,
|
||||
SEED_UPLOAD_NAME_LIKE,
|
||||
};
|
||||
@@ -0,0 +1,31 @@
|
||||
'use strict';
|
||||
|
||||
function getRemotePort() {
|
||||
return (
|
||||
process.env.CLI_TRANSFER_REMOTE_PORT || process.env.CLI_TRANSFER_PULL_REMOTE_PORT || '13710'
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForHttpOk(url, { timeoutMs = 180000 } = {}) {
|
||||
const start = Date.now();
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
try {
|
||||
const res = await fetch(url, { redirect: 'manual' });
|
||||
if (res.status < 500) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
/* not up yet */
|
||||
}
|
||||
if (Date.now() - start > timeoutMs) {
|
||||
throw new Error(`Timed out waiting for ${url}`);
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 400));
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getRemotePort,
|
||||
waitForHttpOk,
|
||||
};
|
||||
@@ -0,0 +1,31 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Helpers for CLI remote transfer e2e (pull-remote / push-remote Jest suites).
|
||||
*
|
||||
* - {@link ./upload-db.js} — SQLite file counts & seed-row signatures (in-process, no child scripts)
|
||||
* - {@link ./seed-media.js} — programmatic seed + CLI shim target
|
||||
* - {@link ./timeouts.js} — runner / Jest timeouts (also required directly from cli-runner)
|
||||
* - {@link ./constants.js} — shared seed filename prefix
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
|
||||
const { countUploadFiles, getSeedUploadSignature } = require('./upload-db');
|
||||
const { seedTransferTestMedia } = require('./seed-media');
|
||||
const { getRemotePort, waitForHttpOk } = require('./http');
|
||||
const timeouts = require('./timeouts');
|
||||
|
||||
const REPO_ROOT = path.join(__dirname, '..', '..', '..');
|
||||
const APP_TEMPLATE_CONSTANTS = path.join(REPO_ROOT, 'tests', 'app-template', 'src', 'constants.js');
|
||||
|
||||
module.exports = {
|
||||
REPO_ROOT,
|
||||
APP_TEMPLATE_CONSTANTS,
|
||||
getRemotePort,
|
||||
waitForHttpOk,
|
||||
countUploadFiles,
|
||||
getSeedUploadSignature,
|
||||
seedTransferTestMedia,
|
||||
...timeouts,
|
||||
};
|
||||
@@ -0,0 +1,99 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
|
||||
const { SEED_UPLOAD_NAME_PREFIX } = require('./constants');
|
||||
|
||||
/**
|
||||
* Deterministic octet stream: every offset has a distinct value (per file index) so a
|
||||
* full-file checksum catches chunk reordering; a single fill byte would not.
|
||||
*/
|
||||
function createDeterministicTransferTestFile(fileIndex, byteLength) {
|
||||
const buf = Buffer.allocUnsafe(byteLength);
|
||||
for (let j = 0; j < byteLength; j += 1) {
|
||||
const mixed = (fileIndex + 1) * 0x9e3779b1 + j * 0x517cc1b7;
|
||||
buf[j] = (mixed ^ (mixed >>> 11) ^ (j << 3)) & 255;
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
function parseCountEnv() {
|
||||
return Math.max(0, parseInt(process.env.TRANSFER_CLI_MEDIA_COUNT || '2', 10));
|
||||
}
|
||||
|
||||
function parseBytesEnv() {
|
||||
return Math.max(1, parseInt(process.env.TRANSFER_CLI_MEDIA_BYTES || '2048', 10));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} appPath - Strapi app root
|
||||
* @param {{ count?: number, bytes?: number }} [options] - defaults from env TRANSFER_CLI_MEDIA_*
|
||||
*/
|
||||
async function seedTransferTestMedia(appPath, options = {}) {
|
||||
const count = options.count ?? parseCountEnv();
|
||||
const bytes = options.bytes ?? parseBytesEnv();
|
||||
|
||||
const { createStrapi } = require('@strapi/strapi');
|
||||
const { CUSTOM_TRANSFER_TOKEN_ACCESS_KEY } = require(path.join(appPath, 'src', 'constants.js'));
|
||||
|
||||
const strapi = createStrapi({ appDir: appPath, distDir: appPath });
|
||||
await strapi.load();
|
||||
|
||||
const { token: transferTokenService } = strapi.service('admin::transfer');
|
||||
const existing = await transferTokenService.list();
|
||||
for (const t of existing) {
|
||||
await transferTokenService.revoke(t.id);
|
||||
}
|
||||
await transferTokenService.create({
|
||||
name: 'CliTransferTestToken',
|
||||
description: 'CLI remote transfer e2e',
|
||||
lifespan: null,
|
||||
permissions: ['push', 'pull'],
|
||||
accessKey: CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
|
||||
});
|
||||
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'strapi-cli-transfer-seed-'));
|
||||
try {
|
||||
for (let i = 0; i < count; i += 1) {
|
||||
const name = `${SEED_UPLOAD_NAME_PREFIX}${i}.bin`;
|
||||
const tmpPath = path.join(tmpDir, name);
|
||||
fs.writeFileSync(tmpPath, createDeterministicTransferTestFile(i, bytes));
|
||||
|
||||
await strapi
|
||||
.plugin('upload')
|
||||
.service('upload')
|
||||
.upload({
|
||||
data: {
|
||||
fileInfo: { name },
|
||||
},
|
||||
files: [
|
||||
{
|
||||
filepath: tmpPath,
|
||||
originalFilename: name,
|
||||
mimetype: 'application/octet-stream',
|
||||
size: bytes,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await strapi.destroy();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
return { count, bytes };
|
||||
}
|
||||
|
||||
/** CLI entry: `node seed-cli-transfer-media.js [appPath]` */
|
||||
async function runFromCli(argv) {
|
||||
const appPath = argv[2] || process.cwd();
|
||||
const { count, bytes } = await seedTransferTestMedia(appPath);
|
||||
console.log(JSON.stringify({ ok: true, count, bytes }));
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
seedTransferTestMedia,
|
||||
runFromCli,
|
||||
};
|
||||
@@ -0,0 +1,68 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Jest / execa timeouts for remote transfer CLI e2e (scale with TRANSFER_CLI_MEDIA_*).
|
||||
* Override: CLI_TRANSFER_REMOTE_*_TIMEOUT_MS (legacy CLI_TRANSFER_PULL_* for runner/jest).
|
||||
*/
|
||||
|
||||
const parseNonNegativeInt = (value, fallback) => {
|
||||
const n = parseInt(value ?? '', 10);
|
||||
return Number.isFinite(n) && n >= 0 ? n : fallback;
|
||||
};
|
||||
|
||||
const totalSeededMediaBytes = () => {
|
||||
const bytes = parseNonNegativeInt(process.env.TRANSFER_CLI_MEDIA_BYTES, 2048);
|
||||
const count = parseNonNegativeInt(process.env.TRANSFER_CLI_MEDIA_COUNT, 2);
|
||||
return bytes * count;
|
||||
};
|
||||
|
||||
const STRAPI_DOMAIN_DEFAULT_RUNNER_MS = 30 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Outer execa(Jest) budget. Non-strapi CLI domains stay at 2 minutes.
|
||||
* @param {string} [domainName]
|
||||
*/
|
||||
function runnerTimeoutMs(domainName) {
|
||||
const explicit =
|
||||
process.env.CLI_TRANSFER_REMOTE_RUNNER_TIMEOUT_MS ||
|
||||
process.env.CLI_TRANSFER_PULL_RUNNER_TIMEOUT_MS;
|
||||
if (explicit) {
|
||||
return parseNonNegativeInt(explicit, STRAPI_DOMAIN_DEFAULT_RUNNER_MS);
|
||||
}
|
||||
|
||||
if (domainName !== 'strapi') {
|
||||
return 2 * 60 * 1000;
|
||||
}
|
||||
|
||||
const total = totalSeededMediaBytes();
|
||||
if (total > 100 * 1024 * 1024) {
|
||||
return 4 * 60 * 60 * 1000;
|
||||
}
|
||||
if (total > 10 * 1024 * 1024) {
|
||||
return 90 * 60 * 1000;
|
||||
}
|
||||
return STRAPI_DOMAIN_DEFAULT_RUNNER_MS;
|
||||
}
|
||||
|
||||
function jestSuiteTimeoutMs() {
|
||||
const explicit =
|
||||
process.env.CLI_TRANSFER_REMOTE_JEST_TIMEOUT_MS ||
|
||||
process.env.CLI_TRANSFER_PULL_JEST_TIMEOUT_MS;
|
||||
if (explicit) {
|
||||
return parseNonNegativeInt(explicit, 10 * 60 * 1000);
|
||||
}
|
||||
const total = totalSeededMediaBytes();
|
||||
if (total > 100 * 1024 * 1024) {
|
||||
return 4 * 60 * 60 * 1000;
|
||||
}
|
||||
if (total > 10 * 1024 * 1024) {
|
||||
return 90 * 60 * 1000;
|
||||
}
|
||||
return 10 * 60 * 1000;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
totalSeededMediaBytes,
|
||||
runnerTimeoutMs,
|
||||
jestSuiteTimeoutMs,
|
||||
};
|
||||
@@ -0,0 +1,59 @@
|
||||
'use strict';
|
||||
|
||||
const path = require('path');
|
||||
const { SEED_UPLOAD_NAME_LIKE } = require('./constants');
|
||||
|
||||
function loadBetterSqlite3(appPath) {
|
||||
try {
|
||||
// Test apps ship the native module; resolve from app first for correct binary.
|
||||
return require(path.join(appPath, 'node_modules', 'better-sqlite3'));
|
||||
} catch {
|
||||
return require('better-sqlite3');
|
||||
}
|
||||
}
|
||||
|
||||
function openUploadsDbReadonly(appPath) {
|
||||
const Database = loadBetterSqlite3(appPath);
|
||||
const dbPath = path.join(appPath, '.tmp', 'data.db');
|
||||
return new Database(dbPath, { readonly: true });
|
||||
}
|
||||
|
||||
function countUploadFiles(appPath) {
|
||||
const db = openUploadsDbReadonly(appPath);
|
||||
try {
|
||||
const row = db.prepare('SELECT COUNT(*) AS c FROM files').get();
|
||||
return row.c;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {{ files: { name: string, hash: string, size: number }[] }}
|
||||
*/
|
||||
function getSeedUploadSignature(appPath) {
|
||||
const db = openUploadsDbReadonly(appPath);
|
||||
try {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT name, hash, size FROM files
|
||||
WHERE name LIKE ?
|
||||
ORDER BY name ASC`
|
||||
)
|
||||
.all(SEED_UPLOAD_NAME_LIKE);
|
||||
return {
|
||||
files: rows.map((r) => ({
|
||||
name: r.name,
|
||||
hash: r.hash,
|
||||
size: Number(r.size),
|
||||
})),
|
||||
};
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
countUploadFiles,
|
||||
getSeedUploadSignature,
|
||||
};
|
||||
@@ -3,14 +3,15 @@
|
||||
const execa = require('execa');
|
||||
|
||||
/**
|
||||
* Run Playwright test command
|
||||
* Run Playwright test command.
|
||||
* Only pass runner-owned env keys; execa merges with process.env by default (extendEnv: true).
|
||||
*/
|
||||
const runPlaywright = async ({ configPath, cwd, port, testAppPath, testArgs }) => {
|
||||
await execa('yarn', ['playwright', 'test', '--config', configPath, ...testArgs], {
|
||||
stdio: 'inherit',
|
||||
cwd,
|
||||
env: {
|
||||
PORT: port,
|
||||
PORT: String(port),
|
||||
HOST: '127.0.0.1',
|
||||
TEST_APP_PATH: testAppPath,
|
||||
STRAPI_DISABLE_EE: !process.env.STRAPI_LICENSE,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
const path = require('path');
|
||||
const execa = require('execa');
|
||||
const { runnerTimeoutMs } = require('../cli-transfer-remote-e2e/timeouts');
|
||||
|
||||
/**
|
||||
* Load domain-specific configuration
|
||||
@@ -50,14 +51,59 @@ const calculateTestAppsRequired = (domainConfigs, concurrency) => {
|
||||
};
|
||||
|
||||
/**
|
||||
* Run Jest test command
|
||||
* The test runner parses CLI flags with yargs before invoking Jest. Unknown options (e.g.
|
||||
* `--testPathPattern=…`) become plain properties on the parsed object and are omitted from `_`,
|
||||
* so they never reached Jest unless the user put them after `--`. Re-serialize those properties
|
||||
* as argv fragments for Jest.
|
||||
*
|
||||
* @param {Record<string, unknown>} testYargs - yargs parse() result for args after `--type`
|
||||
* @returns {string[]}
|
||||
*/
|
||||
const runCLI = async ({ domainDir, jestConfigPath, testApps, testArgs }) => {
|
||||
const env = {
|
||||
TEST_APPS: testApps.join(','),
|
||||
JWT_SECRET: 'test-jwt-secret',
|
||||
};
|
||||
const buildForwardedRunnerArgs = (testYargs) => {
|
||||
/** Keys owned by tests/scripts/run-tests.js (not for Jest). */
|
||||
const runnerKeys = new Set([
|
||||
'_',
|
||||
'$0',
|
||||
'concurrency',
|
||||
'c',
|
||||
'domains',
|
||||
'd',
|
||||
'setup',
|
||||
'f',
|
||||
'updateSnapshot',
|
||||
'u',
|
||||
]);
|
||||
|
||||
const args = [...testYargs._];
|
||||
|
||||
for (const key of Object.keys(testYargs)) {
|
||||
if (runnerKeys.has(key) || key.startsWith('$')) {
|
||||
continue;
|
||||
}
|
||||
const value = testYargs[key];
|
||||
if (value === undefined || value === false) {
|
||||
continue;
|
||||
}
|
||||
const flag = `--${key}`;
|
||||
if (value === true) {
|
||||
args.push(flag);
|
||||
} else if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
args.push(flag, String(item));
|
||||
}
|
||||
} else {
|
||||
args.push(`${flag}=${String(value)}`);
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
};
|
||||
|
||||
/**
|
||||
* Run Jest test command
|
||||
* @param {{ domainDir: string, jestConfigPath: string, testApps: string[], testArgs: string[], domain?: string }} opts
|
||||
*/
|
||||
const runCLI = async ({ domainDir, jestConfigPath, testApps, testArgs, domain }) => {
|
||||
await execa(
|
||||
'jest',
|
||||
[
|
||||
@@ -73,8 +119,14 @@ const runCLI = async ({ domainDir, jestConfigPath, testApps, testArgs }) => {
|
||||
{
|
||||
stdio: 'inherit',
|
||||
cwd: domainDir, // run from the domain directory
|
||||
env, // pass it our custom env values
|
||||
timeout: 5 * 60 * 1000, // 5 minutes
|
||||
// Only set what the runner owns; execa merges with process.env by default (extendEnv: true),
|
||||
// so e.g. TRANSFER_CLI_MEDIA_* from the shell still reach Jest.
|
||||
env: {
|
||||
TEST_APPS: testApps.join(','),
|
||||
JWT_SECRET: process.env.JWT_SECRET || 'test-jwt-secret',
|
||||
},
|
||||
// strapi domain includes remote transfer e2e; longer budget than other domains (see cli-transfer-remote-e2e/timeouts.js).
|
||||
timeout: runnerTimeoutMs(domain),
|
||||
}
|
||||
);
|
||||
};
|
||||
@@ -82,5 +134,6 @@ const runCLI = async ({ domainDir, jestConfigPath, testApps, testArgs }) => {
|
||||
module.exports = {
|
||||
loadDomainConfigs,
|
||||
calculateTestAppsRequired,
|
||||
buildForwardedRunnerArgs,
|
||||
runCLI,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* CLI shim for programmatic seeding — implementation lives in cli-transfer-remote-e2e/seed-media.js.
|
||||
*
|
||||
* node tests/utils/seed-cli-transfer-media.js [appPath]
|
||||
*
|
||||
* Env: TRANSFER_CLI_MEDIA_COUNT, TRANSFER_CLI_MEDIA_BYTES (see tests/cli/README.md)
|
||||
*/
|
||||
|
||||
require('./cli-transfer-remote-e2e/seed-media')
|
||||
.runFromCli(process.argv)
|
||||
.catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user