Files
Ben Irvin bac520e9f8 fix(data-transfer): fix large transfer crashes; show transfer progress (#23479)
* enhancement: update progress per chunk

* fix: check stageprogress exists to make ts happy

* chore: split progress tracker into two methods

* test: fix lint

* enhancement: display readable times

* fix: speed indication for assets

* fix: restore speed indicator and fix double counting

* chore: clean up code

* fix: data transfer memory leak

* fix: yarn.lock

* chore: fix memory logging

* ci: complex project remote transfer

* enhancement: optimized transfer mode

* test(cli): data transfer and env passthrough

* chore: only send one message warning of legacy mode

* enhancement: show transfer totals and estimated time

* test(cli): fix deterministic transfer test files

* fix: push and pull shared utils backwards compatibility

* fix(data-transfer): extract legacy asset chunk parsing and tighten transfer logging/test coverage

* enhancement: checksum negotiation

* enhancement: show skipped file warnings on client

* fix: transfer diagnostics

* test: fix open handle

* fix: clear stall timeout for assets

* chore: fix misleading comments and variables

* test: fix misleading test

* test: fix typo

* test: make checks deterministic, less flaky

* enhancement(data-transfer): speed up asset totals; widen assets start reply window on remote pull

* fix(data-transfer): harden WebSocket JSON serialization for transfer frames

* fix(data-transfer): more transfer hardening

* test: fix test imports

* fix: await async write

* fix(data-transfer): resolve push transfer deadlock and harden async writes

- Extract createAssetsDestinationWritable so Writable callbacks run before
  uploadStream completes (same WS batch as PassThrough chunks).
- Add writable-async-write (write callback + drain/finished race; avoid hang
  on destroy).
- Wire push/pull, remote-source, file & directory sources to shared write().
- Fire-and-forget pull flush: Promise.resolve(flush).catch(onError); guard
  missing stream inside try.
- Add regression tests (assets writable, writable-async-write, handler checks).

* fix(data-transfer): write push stream batches sequentially

Use a for-loop with await write() instead of Promise.all over msg.data
so non-asset stages respect one in-flight write per objectMode Writable
and backpressure from writable-async-write.

- Validate minChunksForBackpressure in assertReadStreamBackpressure
- Add engine test for non-Buffer asset chunk byte progress (counts as 1)
- Assert push.ts keeps sequential msg.data handling in static handler test

* fix(data-transfer): align push streamAsset with remote-source and harden tests

- Push handler: combine stream/end under one branch, error when start is missing
  or action is invalid; shorten stage write comments.
- Engine version-matching tests: use a fresh createDestination() per engine so
  parallel transfers do not share destination writables (MaxListeners warnings).
- File destination tests: mock createWriteStream with a new Writable per call.
- CLI transfer tests: mock progress.stream so transfer::finish runs after transfer
  and clears the progress setInterval (fixes Jest worker hang).
- Misc test cleanup: assets-destination timeout clearTimeout, collect listeners,
  writable-async-write teardown; tighten push/static test descriptions.

* test: remove parity test

* fix(data-transfer): harden collect() and stabilize transfer tests

- collect(): settle once, remove listeners on resolve/reject, avoid double completion
- engine tests: add expectHeapGrowthWithinNoise for heap smoke checks
- CLI transfer tests: console spies in beforeAll; jest.restoreAllMocks in afterAll
- stream test: remove removeAllListeners workaround
2026-04-20 16:08:05 +02:00

100 lines
3.0 KiB
JavaScript

'use strict';
const fs = require('fs');
const path = require('path');
const os = require('os');
const { SEED_UPLOAD_NAME_PREFIX } = require('./constants');
/**
* Deterministic octet stream: every offset has a distinct value (per file index) so a
* full-file checksum catches chunk reordering; a single fill byte would not.
*/
function createDeterministicTransferTestFile(fileIndex, byteLength) {
const buf = Buffer.allocUnsafe(byteLength);
for (let j = 0; j < byteLength; j += 1) {
const mixed = (fileIndex + 1) * 0x9e3779b1 + j * 0x517cc1b7;
buf[j] = (mixed ^ (mixed >>> 11) ^ (j << 3)) & 255;
}
return buf;
}
function parseCountEnv() {
return Math.max(0, parseInt(process.env.TRANSFER_CLI_MEDIA_COUNT || '2', 10));
}
function parseBytesEnv() {
return Math.max(1, parseInt(process.env.TRANSFER_CLI_MEDIA_BYTES || '2048', 10));
}
/**
* @param {string} appPath - Strapi app root
* @param {{ count?: number, bytes?: number }} [options] - defaults from env TRANSFER_CLI_MEDIA_*
*/
async function seedTransferTestMedia(appPath, options = {}) {
const count = options.count ?? parseCountEnv();
const bytes = options.bytes ?? parseBytesEnv();
const { createStrapi } = require('@strapi/strapi');
const { CUSTOM_TRANSFER_TOKEN_ACCESS_KEY } = require(path.join(appPath, 'src', 'constants.js'));
const strapi = createStrapi({ appDir: appPath, distDir: appPath });
await strapi.load();
const { token: transferTokenService } = strapi.service('admin::transfer');
const existing = await transferTokenService.list();
for (const t of existing) {
await transferTokenService.revoke(t.id);
}
await transferTokenService.create({
name: 'CliTransferTestToken',
description: 'CLI remote transfer e2e',
lifespan: null,
permissions: ['push', 'pull'],
accessKey: CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
});
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'strapi-cli-transfer-seed-'));
try {
for (let i = 0; i < count; i += 1) {
const name = `${SEED_UPLOAD_NAME_PREFIX}${i}.bin`;
const tmpPath = path.join(tmpDir, name);
fs.writeFileSync(tmpPath, createDeterministicTransferTestFile(i, bytes));
await strapi
.plugin('upload')
.service('upload')
.upload({
data: {
fileInfo: { name },
},
files: [
{
filepath: tmpPath,
originalFilename: name,
mimetype: 'application/octet-stream',
size: bytes,
},
],
});
}
} finally {
await strapi.destroy();
fs.rmSync(tmpDir, { recursive: true, force: true });
}
return { count, bytes };
}
/** CLI entry: `node seed-cli-transfer-media.js [appPath]` */
async function runFromCli(argv) {
const appPath = argv[2] || process.cwd();
const { count, bytes } = await seedTransferTestMedia(appPath);
console.log(JSON.stringify({ ok: true, count, bytes }));
}
module.exports = {
seedTransferTestMedia,
runFromCli,
};