diff --git a/src/testRunner/externalCompileRunner.ts b/src/testRunner/externalCompileRunner.ts deleted file mode 100644 index 92acf6532b0..00000000000 --- a/src/testRunner/externalCompileRunner.ts +++ /dev/null @@ -1,323 +0,0 @@ -namespace Harness { - const fs: typeof import("fs") = require("fs"); - const path: typeof import("path") = require("path"); - const del: typeof import("del") = require("del"); - - interface ExecResult { - stdout: Buffer; - stderr: Buffer; - status: number | null; - } - - interface UserConfig { - types: string[]; - cloneUrl: string; - branch?: string; - path?: string; - } - - abstract class ExternalCompileRunnerBase extends RunnerBase { - abstract testDir: string; - abstract report(result: ExecResult): string | null; - enumerateTestFiles() { - return IO.getDirectories(this.testDir); - } - /** Setup the runner's tests so that they are ready to be executed by the harness - * The first test should be a describe/it block that sets up the harness's compiler instance appropriately - */ - initializeTests(): void { - // Read in and evaluate the test list - const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); - - // eslint-disable-next-line @typescript-eslint/no-this-alias - const cls = this; - describe(`${this.kind()} code samples`, function (this: Mocha.Suite) { - this.timeout(600_000); // 10 minutes - for (const test of testList) { - cls.runTest(typeof test === "string" ? test : test.file); - } - }); - } - private runTest(directoryName: string) { - // eslint-disable-next-line @typescript-eslint/no-this-alias - const cls = this; - const timeout = 600_000; // 10 minutes - describe(directoryName, function (this: Mocha.Suite) { - this.timeout(timeout); - const cp: typeof import("child_process") = require("child_process"); - - it("should build successfully", () => { - let cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directoryName); - const originalCwd = cwd; - const stdio = isWorker ? "pipe" : "inherit"; - let types: string[] | undefined; - if (fs.existsSync(path.join(cwd, "test.json"))) { - const config = JSON.parse(fs.readFileSync(path.join(cwd, "test.json"), { encoding: "utf8" })) as UserConfig; - ts.Debug.assert(!!config.types, "Bad format from test.json: Types field must be present."); - ts.Debug.assert(!!config.cloneUrl, "Bad format from test.json: cloneUrl field must be present."); - const submoduleDir = path.join(cwd, directoryName); - if (!fs.existsSync(submoduleDir)) { - exec("git", ["--work-tree", submoduleDir, "clone", "-b", config.branch || "master", config.cloneUrl, path.join(submoduleDir, ".git")], { cwd }); - } - else { - exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "checkout", config.branch || "master"], { cwd: submoduleDir }); - exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "reset", "HEAD", "--hard"], { cwd: submoduleDir }); - exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "clean", "-f"], { cwd: submoduleDir }); - exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "pull", "-f"], { cwd: submoduleDir }); - } - - types = config.types; - - cwd = config.path ? path.join(cwd, config.path) : submoduleDir; - } - const npmVersionText = exec("npm", ["--version"], { cwd, stdio: "pipe" })?.trim(); - const npmVersion = npmVersionText ? ts.Version.tryParse(npmVersionText.trim()) : undefined; - const isV7OrLater = !!npmVersion && npmVersion.major >= 7; - if (fs.existsSync(path.join(cwd, "package.json"))) { - if (fs.existsSync(path.join(cwd, "package-lock.json"))) { - fs.unlinkSync(path.join(cwd, "package-lock.json")); - } - if (fs.existsSync(path.join(cwd, "node_modules"))) { - del.sync(path.join(cwd, "node_modules"), { force: true }); - } - exec("npm", ["i", "--ignore-scripts", ...(isV7OrLater ? ["--legacy-peer-deps"] : [])], { cwd, timeout: timeout / 2 }); // NPM shouldn't take the entire timeout - if it takes a long time, it should be terminated and we should log the failure - } - const args = [path.join(IO.getWorkspaceRoot(), "built/local/tsc.js")]; - if (types) { - args.push("--types", types.join(",")); - // Also actually install those types (for, eg, the js projects which need node) - if (types.length) { - exec("npm", ["i", ...types.map(t => `@types/${t}`), "--no-save", "--ignore-scripts", ...(isV7OrLater ? ["--legacy-peer-deps"] : [])], { cwd: originalCwd, timeout: timeout / 2 }); // NPM shouldn't take the entire timeout - if it takes a long time, it should be terminated and we should log the failure - } - } - args.push("--noEmit"); - Baseline.runBaseline(`${cls.kind()}/${directoryName}.log`, cls.report(cp.spawnSync(`node`, args, { cwd, timeout, shell: true }))); - - function exec(command: string, args: string[], options: { cwd: string, timeout?: number, stdio?: import("child_process").StdioOptions }): string | undefined { - const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { shell: true, stdio, ...options }); - if (res.status !== 0) { - throw new Error(`${command} ${args.join(" ")} for ${directoryName} failed: ${res.stdout && res.stdout.toString()}`); - } - return options.stdio === "pipe" ? res.stdout.toString("utf8") : undefined; - } - }); - }); - } - } - - export class UserCodeRunner extends ExternalCompileRunnerBase { - readonly testDir = "tests/cases/user/"; - kind(): TestRunnerKind { - return "user"; - } - report(result: ExecResult) { - // eslint-disable-next-line no-null/no-null - return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} -Standard output: -${sortErrors(stripAbsoluteImportPaths(result.stdout.toString().replace(/\r\n/g, "\n")))} - - -Standard error: -${stripAbsoluteImportPaths(result.stderr.toString().replace(/\r\n/g, "\n"))}`; - } - } - - export class DockerfileRunner extends ExternalCompileRunnerBase { - readonly testDir = "tests/cases/docker/"; - kind(): TestRunnerKind { - return "docker"; - } - initializeTests(): void { - // Read in and evaluate the test list - const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); - - // eslint-disable-next-line @typescript-eslint/no-this-alias - const cls = this; - describe(`${this.kind()} code samples`, function (this: Mocha.Suite) { - this.timeout(cls.timeout); // 20 minutes - before(() => { - cls.exec("docker", ["build", ".", "-t", "typescript/typescript"], { cwd: IO.getWorkspaceRoot() }); // cached because workspace is hashed to determine cacheability - }); - for (const test of testList) { - const directory = typeof test === "string" ? test : test.file; - const cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directory); - it(`should build ${directory} successfully`, () => { - const imageName = `tstest/${directory}`; - cls.exec("docker", ["build", "--no-cache", ".", "-t", imageName], { cwd }); // --no-cache so the latest version of the repos referenced is always fetched - const cp: typeof import("child_process") = require("child_process"); - Baseline.runBaseline(`${cls.kind()}/${directory}.log`, cls.report(cp.spawnSync(`docker`, ["run", imageName], { cwd, timeout: cls.timeout, shell: true }))); - }); - } - }); - } - - private timeout = 1_200_000; // 20 minutes; - private exec(command: string, args: string[], options: { cwd: string }): void { - const cp: typeof import("child_process") = require("child_process"); - const stdio = isWorker ? "pipe" : "inherit"; - const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { timeout: this.timeout, shell: true, stdio, ...options }); - if (res.status !== 0) { - throw new Error(`${command} ${args.join(" ")} for ${options.cwd} failed: ${res.stdout && res.stdout.toString()}`); - } - } - report(result: ExecResult) { - // eslint-disable-next-line no-null/no-null - return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} -Standard output: -${sanitizeDockerfileOutput(result.stdout.toString())} - - -Standard error: -${sanitizeDockerfileOutput(result.stderr.toString())}`; - } - } - - function sanitizeDockerfileOutput(result: string): string { - return [ - normalizeNewlines, - stripANSIEscapes, - stripRushStageNumbers, - stripWebpackHash, - sanitizeVersionSpecifiers, - sanitizeTimestamps, - sanitizeSizes, - sanitizeUnimportantGulpOutput, - stripAbsoluteImportPaths, - ].reduce((result, f) => f(result), result); - } - - function normalizeNewlines(result: string): string { - return result.replace(/\r\n/g, "\n"); - } - - function stripANSIEscapes(result: string): string { - return result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, ""); - } - - function stripRushStageNumbers(result: string): string { - return result.replace(/\d+ of \d+:/g, "XX of XX:"); - } - - function stripWebpackHash(result: string): string { - return result.replace(/Hash: \w+/g, "Hash: [redacted]"); - } - - function sanitizeSizes(result: string): string { - return result.replace(/\d+(\.\d+)? ((Ki|M)B|bytes)/g, "X KiB"); - } - - /** - * Gulp's output order within a `parallel` block is nondeterministic (and there's no way to configure it to execute in series), - * so we purge as much of the gulp output as we can - */ - function sanitizeUnimportantGulpOutput(result: string): string { - return result.replace(/^.*(\] (Starting)|(Finished)).*$/gm, "") // "gulp" task start/end messages (nondeterministic order) - .replace(/^.*(\] . (finished)|(started)).*$/gm, "") // "just" task start/end messages (nondeterministic order) - .replace(/^.*\] Respawned to PID: \d+.*$/gm, "") // PID of child is OS and system-load dependent (likely stableish in a container but still dangerous) - .replace(/\n+/g, "\n") - .replace(/\/tmp\/yarn--.*?\/node/g, ""); - } - - function sanitizeTimestamps(result: string): string { - return result.replace(/\[\d?\d:\d\d:\d\d (A|P)M\]/g, "[XX:XX:XX XM]") - .replace(/\[\d?\d:\d\d:\d\d\]/g, "[XX:XX:XX]") - .replace(/\/\d+-\d+-[\d_TZ]+-debug.log/g, "\/XXXX-XX-XXXXXXXXX-debug.log") - .replace(/\d+\/\d+\/\d+ \d+:\d+:\d+ (AM|PM)/g, "XX/XX/XX XX:XX:XX XM") - .replace(/\d+(\.\d+)? sec(onds?)?/g, "? seconds") - .replace(/\d+(\.\d+)? min(utes?)?/g, "") - .replace(/\d+(\.\d+)? ?m?s/g, "?s") - .replace(/ \(\?s\)/g, ""); - } - - function sanitizeVersionSpecifiers(result: string): string { - return result - .replace(/\d+.\d+.\d+-insiders.\d\d\d\d\d\d\d\d/g, "X.X.X-insiders.xxxxxxxx") - .replace(/Rush Multi-Project Build Tool (\d+)\.\d+\.\d+/g, "Rush Multi-Project Build Tool $1.X.X") - .replace(/([@v\()])\d+\.\d+\.\d+/g, "$1X.X.X") - .replace(/webpack (\d+)\.\d+\.\d+/g, "webpack $1.X.X") - .replace(/Webpack version: (\d+)\.\d+\.\d+/g, "Webpack version: $1.X.X"); - } - - /** - * Import types and some other error messages use absolute paths in errors as they have no context to be written relative to; - * This is problematic for error baselines, so we grep for them and strip them out. - */ - function stripAbsoluteImportPaths(result: string) { - const workspaceRegexp = new RegExp(IO.getWorkspaceRoot().replace(/\\/g, "\\\\"), "g"); - return result - .replace(/import\(".*?\/tests\/cases\/user\//g, `import("/`) - .replace(/Module '".*?\/tests\/cases\/user\//g, `Module '"/`) - .replace(workspaceRegexp, "../../.."); - } - - function sortErrors(result: string) { - return ts.flatten(splitBy(result.split("\n"), s => /^\S+/.test(s)).sort(compareErrorStrings)).join("\n"); - } - - const errorRegexp = /^(.+\.[tj]sx?)\((\d+),(\d+)\)(: error TS.*)/; - function compareErrorStrings(a: string[], b: string[]) { - ts.Debug.assertGreaterThanOrEqual(a.length, 1); - ts.Debug.assertGreaterThanOrEqual(b.length, 1); - const matchA = a[0].match(errorRegexp); - if (!matchA) { - return -1; - } - const matchB = b[0].match(errorRegexp); - if (!matchB) { - return 1; - } - const [, errorFileA, lineNumberStringA, columnNumberStringA, remainderA] = matchA; - const [, errorFileB, lineNumberStringB, columnNumberStringB, remainderB] = matchB; - return ts.comparePathsCaseSensitive(errorFileA, errorFileB) || - ts.compareValues(parseInt(lineNumberStringA), parseInt(lineNumberStringB)) || - ts.compareValues(parseInt(columnNumberStringA), parseInt(columnNumberStringB)) || - ts.compareStringsCaseSensitive(remainderA, remainderB) || - ts.compareStringsCaseSensitive(a.slice(1).join("\n"), b.slice(1).join("\n")); - } - - export class DefinitelyTypedRunner extends ExternalCompileRunnerBase { - readonly testDir = "../DefinitelyTyped/types/"; - workingDirectory = this.testDir; - kind(): TestRunnerKind { - return "dt"; - } - report(result: ExecResult) { - // eslint-disable-next-line no-null/no-null - return !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} -Standard output: -${result.stdout.toString().replace(/\r\n/g, "\n")} - - -Standard error: -${result.stderr.toString().replace(/\r\n/g, "\n")}`; - } - } - - /** - * Split an array into multiple arrays whenever `isStart` returns true. - * @example - * splitBy([1,2,3,4,5,6], isOdd) - * ==> [[1, 2], [3, 4], [5, 6]] - * where - * const isOdd = n => !!(n % 2) - */ - function splitBy(xs: T[], isStart: (x: T) => boolean): T[][] { - const result = []; - let group: T[] = []; - for (const x of xs) { - if (isStart(x)) { - if (group.length) { - result.push(group); - } - group = [x]; - } - else { - group.push(x); - } - } - if (group.length) { - result.push(group); - } - return result; - } -} diff --git a/src/testRunner/runner.ts b/src/testRunner/runner.ts index 4427390d7f3..2007f872d97 100644 --- a/src/testRunner/runner.ts +++ b/src/testRunner/runner.ts @@ -58,12 +58,6 @@ ${JSON.stringify(dupes, undefined, 2)}`); return new RWC.RWCRunner(); case "test262": return new Test262BaselineRunner(); - case "user": - return new UserCodeRunner(); - case "dt": - return new DefinitelyTypedRunner(); - case "docker": - return new DockerfileRunner(); } return ts.Debug.fail(`Unknown runner kind ${kind}`); } @@ -201,15 +195,6 @@ ${JSON.stringify(dupes, undefined, 2)}`); case "test262": runners.push(new Test262BaselineRunner()); break; - case "user": - runners.push(new UserCodeRunner()); - break; - case "dt": - runners.push(new DefinitelyTypedRunner()); - break; - case "docker": - runners.push(new DockerfileRunner()); - break; } } } @@ -228,12 +213,6 @@ ${JSON.stringify(dupes, undefined, 2)}`); runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess)); runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server)); // runners.push(new GeneratedFourslashRunner()); - - // CRON-only tests - if (process.env.TRAVIS_EVENT_TYPE === "cron") { - runners.push(new UserCodeRunner()); - runners.push(new DockerfileRunner()); - } } if (runUnitTests === undefined) { runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for diff --git a/src/testRunner/tsconfig.json b/src/testRunner/tsconfig.json index 0f7ee625094..dca0ce05dc6 100644 --- a/src/testRunner/tsconfig.json +++ b/src/testRunner/tsconfig.json @@ -41,7 +41,6 @@ "compilerRunner.ts", "projectsRunner.ts", "rwcRunner.ts", - "externalCompileRunner.ts", "test262Runner.ts", "parallel/host.ts",