mirror of
https://github.com/strapi/strapi.git
synced 2026-05-03 16:22:30 +00:00
bac520e9f8
* enhancement: update progress per chunk * fix: check stageprogress exists to make ts happy * chore: split progress tracker into two methods * test: fix lint * enhancement: display readable times * fix: speed indication for assets * fix: restore speed indicator and fix double counting * chore: clean up code * fix: data transfer memory leak * fix: yarn.lock * chore: fix memory logging * ci: complex project remote transfer * enhancement: optimized transfer mode * test(cli): data transfer and env passthrough * chore: only send one message warning of legacy mode * enhancement: show transfer totals and estimated time * test(cli): fix deterministic transfer test files * fix: push and pull shared utils backwards compatibility * fix(data-transfer): extract legacy asset chunk parsing and tighten transfer logging/test coverage * enhancement: checksum negotiation * enhancement: show skipped file warnings on client * fix: transfer diagnostics * test: fix open handle * fix: clear stall timeout for assets * chore: fix misleading comments and variables * test: fix misleading test * test: fix typo * test: make checks deterministic, less flaky * enhancement(data-transfer): speed up asset totals; widen assets start reply window on remote pull * fix(data-transfer): harden WebSocket JSON serialization for transfer frames * fix(data-transfer): more transfer hardening * test: fix test imports * fix: await async write * fix(data-transfer): resolve push transfer deadlock and harden async writes - Extract createAssetsDestinationWritable so Writable callbacks run before uploadStream completes (same WS batch as PassThrough chunks). - Add writable-async-write (write callback + drain/finished race; avoid hang on destroy). - Wire push/pull, remote-source, file & directory sources to shared write(). - Fire-and-forget pull flush: Promise.resolve(flush).catch(onError); guard missing stream inside try. - Add regression tests (assets writable, writable-async-write, handler checks). * fix(data-transfer): write push stream batches sequentially Use a for-loop with await write() instead of Promise.all over msg.data so non-asset stages respect one in-flight write per objectMode Writable and backpressure from writable-async-write. - Validate minChunksForBackpressure in assertReadStreamBackpressure - Add engine test for non-Buffer asset chunk byte progress (counts as 1) - Assert push.ts keeps sequential msg.data handling in static handler test * fix(data-transfer): align push streamAsset with remote-source and harden tests - Push handler: combine stream/end under one branch, error when start is missing or action is invalid; shorten stage write comments. - Engine version-matching tests: use a fresh createDestination() per engine so parallel transfers do not share destination writables (MaxListeners warnings). - File destination tests: mock createWriteStream with a new Writable per call. - CLI transfer tests: mock progress.stream so transfer::finish runs after transfer and clears the progress setInterval (fixes Jest worker hang). - Misc test cleanup: assets-destination timeout clearTimeout, collect listeners, writable-async-write teardown; tighten push/static test descriptions. * test: remove parity test * fix(data-transfer): harden collect() and stabilize transfer tests - collect(): settle once, remove listeners on resolve/reject, avoid double completion - engine tests: add expectHeapGrowthWithinNoise for heap smoke checks - CLI transfer tests: console spies in beforeAll; jest.restoreAllMocks in afterAll - stream test: remove removeAllListeners workaround
100 lines
3.0 KiB
JavaScript
100 lines
3.0 KiB
JavaScript
'use strict';
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const os = require('os');
|
|
|
|
const { SEED_UPLOAD_NAME_PREFIX } = require('./constants');
|
|
|
|
/**
|
|
* Deterministic octet stream: every offset has a distinct value (per file index) so a
|
|
* full-file checksum catches chunk reordering; a single fill byte would not.
|
|
*/
|
|
function createDeterministicTransferTestFile(fileIndex, byteLength) {
|
|
const buf = Buffer.allocUnsafe(byteLength);
|
|
for (let j = 0; j < byteLength; j += 1) {
|
|
const mixed = (fileIndex + 1) * 0x9e3779b1 + j * 0x517cc1b7;
|
|
buf[j] = (mixed ^ (mixed >>> 11) ^ (j << 3)) & 255;
|
|
}
|
|
return buf;
|
|
}
|
|
|
|
function parseCountEnv() {
|
|
return Math.max(0, parseInt(process.env.TRANSFER_CLI_MEDIA_COUNT || '2', 10));
|
|
}
|
|
|
|
function parseBytesEnv() {
|
|
return Math.max(1, parseInt(process.env.TRANSFER_CLI_MEDIA_BYTES || '2048', 10));
|
|
}
|
|
|
|
/**
|
|
* @param {string} appPath - Strapi app root
|
|
* @param {{ count?: number, bytes?: number }} [options] - defaults from env TRANSFER_CLI_MEDIA_*
|
|
*/
|
|
async function seedTransferTestMedia(appPath, options = {}) {
|
|
const count = options.count ?? parseCountEnv();
|
|
const bytes = options.bytes ?? parseBytesEnv();
|
|
|
|
const { createStrapi } = require('@strapi/strapi');
|
|
const { CUSTOM_TRANSFER_TOKEN_ACCESS_KEY } = require(path.join(appPath, 'src', 'constants.js'));
|
|
|
|
const strapi = createStrapi({ appDir: appPath, distDir: appPath });
|
|
await strapi.load();
|
|
|
|
const { token: transferTokenService } = strapi.service('admin::transfer');
|
|
const existing = await transferTokenService.list();
|
|
for (const t of existing) {
|
|
await transferTokenService.revoke(t.id);
|
|
}
|
|
await transferTokenService.create({
|
|
name: 'CliTransferTestToken',
|
|
description: 'CLI remote transfer e2e',
|
|
lifespan: null,
|
|
permissions: ['push', 'pull'],
|
|
accessKey: CUSTOM_TRANSFER_TOKEN_ACCESS_KEY,
|
|
});
|
|
|
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'strapi-cli-transfer-seed-'));
|
|
try {
|
|
for (let i = 0; i < count; i += 1) {
|
|
const name = `${SEED_UPLOAD_NAME_PREFIX}${i}.bin`;
|
|
const tmpPath = path.join(tmpDir, name);
|
|
fs.writeFileSync(tmpPath, createDeterministicTransferTestFile(i, bytes));
|
|
|
|
await strapi
|
|
.plugin('upload')
|
|
.service('upload')
|
|
.upload({
|
|
data: {
|
|
fileInfo: { name },
|
|
},
|
|
files: [
|
|
{
|
|
filepath: tmpPath,
|
|
originalFilename: name,
|
|
mimetype: 'application/octet-stream',
|
|
size: bytes,
|
|
},
|
|
],
|
|
});
|
|
}
|
|
} finally {
|
|
await strapi.destroy();
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
}
|
|
|
|
return { count, bytes };
|
|
}
|
|
|
|
/** CLI entry: `node seed-cli-transfer-media.js [appPath]` */
|
|
async function runFromCli(argv) {
|
|
const appPath = argv[2] || process.cwd();
|
|
const { count, bytes } = await seedTransferTestMedia(appPath);
|
|
console.log(JSON.stringify({ ok: true, count, bytes }));
|
|
}
|
|
|
|
module.exports = {
|
|
seedTransferTestMedia,
|
|
runFromCli,
|
|
};
|