Files
Ben Irvin bac520e9f8 fix(data-transfer): fix large transfer crashes; show transfer progress (#23479)
* enhancement: update progress per chunk

* fix: check stageprogress exists to make ts happy

* chore: split progress tracker into two methods

* test: fix lint

* enhancement: display readable times

* fix: speed indication for assets

* fix: restore speed indicator and fix double counting

* chore: clean up code

* fix: data transfer memory leak

* fix: yarn.lock

* chore: fix memory logging

* ci: complex project remote transfer

* enhancement: optimized transfer mode

* test(cli): data transfer and env passthrough

* chore: only send one message warning of legacy mode

* enhancement: show transfer totals and estimated time

* test(cli): fix deterministic transfer test files

* fix: push and pull shared utils backwards compatibility

* fix(data-transfer): extract legacy asset chunk parsing and tighten transfer logging/test coverage

* enhancement: checksum negotiation

* enhancement: show skipped file warnings on client

* fix: transfer diagnostics

* test: fix open handle

* fix: clear stall timeout for assets

* chore: fix misleading comments and variables

* test: fix misleading test

* test: fix typo

* test: make checks deterministic, less flaky

* enhancement(data-transfer): speed up asset totals; widen assets start reply window on remote pull

* fix(data-transfer): harden WebSocket JSON serialization for transfer frames

* fix(data-transfer): more transfer hardening

* test: fix test imports

* fix: await async write

* fix(data-transfer): resolve push transfer deadlock and harden async writes

- Extract createAssetsDestinationWritable so Writable callbacks run before
  uploadStream completes (same WS batch as PassThrough chunks).
- Add writable-async-write (write callback + drain/finished race; avoid hang
  on destroy).
- Wire push/pull, remote-source, file & directory sources to shared write().
- Fire-and-forget pull flush: Promise.resolve(flush).catch(onError); guard
  missing stream inside try.
- Add regression tests (assets writable, writable-async-write, handler checks).

* fix(data-transfer): write push stream batches sequentially

Use a for-loop with await write() instead of Promise.all over msg.data
so non-asset stages respect one in-flight write per objectMode Writable
and backpressure from writable-async-write.

- Validate minChunksForBackpressure in assertReadStreamBackpressure
- Add engine test for non-Buffer asset chunk byte progress (counts as 1)
- Assert push.ts keeps sequential msg.data handling in static handler test

* fix(data-transfer): align push streamAsset with remote-source and harden tests

- Push handler: combine stream/end under one branch, error when start is missing
  or action is invalid; shorten stage write comments.
- Engine version-matching tests: use a fresh createDestination() per engine so
  parallel transfers do not share destination writables (MaxListeners warnings).
- File destination tests: mock createWriteStream with a new Writable per call.
- CLI transfer tests: mock progress.stream so transfer::finish runs after transfer
  and clears the progress setInterval (fixes Jest worker hang).
- Misc test cleanup: assets-destination timeout clearTimeout, collect listeners,
  writable-async-write teardown; tighten push/static test descriptions.

* test: remove parity test

* fix(data-transfer): harden collect() and stabilize transfer tests

- collect(): settle once, remove listeners on resolve/reject, avoid double completion
- engine tests: add expectHeapGrowthWithinNoise for heap smoke checks
- CLI transfer tests: console spies in beforeAll; jest.restoreAllMocks in afterAll
- stream test: remove removeAllListeners workaround
2026-04-20 16:08:05 +02:00

140 lines
3.9 KiB
JavaScript

'use strict';
const path = require('path');
const execa = require('execa');
const { runnerTimeoutMs } = require('../cli-transfer-remote-e2e/timeouts');
/**
* Load domain-specific configuration
*/
const loadDomainConfigs = async (testsDir, domains, argv) => {
const fs = require('node:fs/promises');
const loadDomainConfig = async (domain) => {
try {
const configPath = path.join(testsDir, domain, 'config.js');
await fs.access(configPath);
// Import config.js and call it as a function
const config = require(configPath);
if (typeof config === 'function') {
return await config(argv);
}
return config;
} catch (e) {
// use default config
return {
testApps: 1,
};
}
};
// Load the domain configs into an object with keys of the name of the test domain
const domainConfigs = {};
await Promise.all(
domains.map(async (domain) => {
domainConfigs[domain] = await loadDomainConfig(domain);
})
);
return domainConfigs;
};
/**
* Calculate the number of test apps required based on domain configs
*/
const calculateTestAppsRequired = (domainConfigs, concurrency) => {
// Determine the number of simultaneous test apps we need by taking the concurrency number of highest testApps requested from config
return Object.entries(domainConfigs)
.map(([, value]) => value.testApps) // Extract testApps values from config
.sort((a, b) => b - a) // Sort in descending order
.slice(0, concurrency) // Take the top X values
.reduce((acc, value) => acc + value, 0); // Sum up the values
};
/**
* The test runner parses CLI flags with yargs before invoking Jest. Unknown options (e.g.
* `--testPathPattern=…`) become plain properties on the parsed object and are omitted from `_`,
* so they never reached Jest unless the user put them after `--`. Re-serialize those properties
* as argv fragments for Jest.
*
* @param {Record<string, unknown>} testYargs - yargs parse() result for args after `--type`
* @returns {string[]}
*/
const buildForwardedRunnerArgs = (testYargs) => {
/** Keys owned by tests/scripts/run-tests.js (not for Jest). */
const runnerKeys = new Set([
'_',
'$0',
'concurrency',
'c',
'domains',
'd',
'setup',
'f',
'updateSnapshot',
'u',
]);
const args = [...testYargs._];
for (const key of Object.keys(testYargs)) {
if (runnerKeys.has(key) || key.startsWith('$')) {
continue;
}
const value = testYargs[key];
if (value === undefined || value === false) {
continue;
}
const flag = `--${key}`;
if (value === true) {
args.push(flag);
} else if (Array.isArray(value)) {
for (const item of value) {
args.push(flag, String(item));
}
} else {
args.push(`${flag}=${String(value)}`);
}
}
return args;
};
/**
* Run Jest test command
* @param {{ domainDir: string, jestConfigPath: string, testApps: string[], testArgs: string[], domain?: string }} opts
*/
const runCLI = async ({ domainDir, jestConfigPath, testApps, testArgs, domain }) => {
await execa(
'jest',
[
'--config',
jestConfigPath,
'--rootDir',
domainDir,
'--color',
'--verbose',
'--runInBand', // tests must not run concurrently
...testArgs,
],
{
stdio: 'inherit',
cwd: domainDir, // run from the domain directory
// Only set what the runner owns; execa merges with process.env by default (extendEnv: true),
// so e.g. TRANSFER_CLI_MEDIA_* from the shell still reach Jest.
env: {
TEST_APPS: testApps.join(','),
JWT_SECRET: process.env.JWT_SECRET || 'test-jwt-secret',
},
// strapi domain includes remote transfer e2e; longer budget than other domains (see cli-transfer-remote-e2e/timeouts.js).
timeout: runnerTimeoutMs(domain),
}
);
};
module.exports = {
loadDomainConfigs,
calculateTestAppsRequired,
buildForwardedRunnerArgs,
runCLI,
};