From a5fa75a0cd095702d41e259cd2eb73d2c777b162 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Thu, 9 Nov 2017 14:21:08 -0800 Subject: [PATCH 1/3] DefinitelyTypedRunner skips ExpectErrors If all errors were ExpectErrors, then it does not fail. --- src/harness/externalCompileRunner.ts | 91 +++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/src/harness/externalCompileRunner.ts b/src/harness/externalCompileRunner.ts index 9b8fc8c4fc5..ec8acf4872e 100644 --- a/src/harness/externalCompileRunner.ts +++ b/src/harness/externalCompileRunner.ts @@ -1,14 +1,17 @@ /// /// +const fs = require("fs"); +const path = require("path"); abstract class ExternalCompileRunnerBase extends RunnerBase { abstract testDir: string; - public enumerateTestFiles() { + abstract report(result: any, cwd: string): string; + enumerateTestFiles() { return Harness.IO.getDirectories(this.testDir); } /** Setup the runner's tests so that they are ready to be executed by the harness * The first test should be a describe/it block that sets up the harness's compiler instance appropriately */ - public initializeTests(): void { + initializeTests(): void { // Read in and evaluate the test list const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles(); @@ -36,15 +39,7 @@ abstract class ExternalCompileRunnerBase extends RunnerBase { if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`); } Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => { - const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }); - // tslint:disable-next-line:no-null-keyword - return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} -Standard output: -${result.stdout.toString().replace(/\r\n/g, "\n")} - - -Standard error: -${result.stderr.toString().replace(/\r\n/g, "\n")}`; + return this.report(cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }), cwd); }); }); }); @@ -52,16 +47,80 @@ ${result.stderr.toString().replace(/\r\n/g, "\n")}`; } class UserCodeRunner extends ExternalCompileRunnerBase { - public readonly testDir = "tests/cases/user/"; - public kind(): TestRunnerKind { + readonly testDir = "tests/cases/user/"; + kind(): TestRunnerKind { return "user"; } + report(result: any) { + // tslint:disable-next-line:no-null-keyword + return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} +Standard output: +${result.stdout.toString().replace(/\r\n/g, "\n")} + + +Standard error: +${result.stderr.toString().replace(/\r\n/g, "\n")}`; + } } class DefinitelyTypedRunner extends ExternalCompileRunnerBase { - public readonly testDir = "../DefinitelyTyped/types/"; - public workingDirectory = this.testDir; - public kind(): TestRunnerKind { + readonly testDir = "../DefinitelyTyped/types/"; + workingDirectory = this.testDir; + kind(): TestRunnerKind { return "dt"; } + report(result: any, cwd: string) { + const stdout = filterExpectedErrors(result.stdout.toString(), cwd) + const stderr = result.stderr.toString() + // tslint:disable-next-line:no-null-keyword + return !stdout.length && !stderr.length ? null : `Exit Code: ${result.status} +Standard output: +${stdout.replace(/\r\n/g, "\n")} + + +Standard error: +${stderr.replace(/\r\n/g, "\n")}`; + } +} + +function filterExpectedErrors(errors: string, cwd: string): string { + return breaks(errors.split("\n"), s => /^\w+/.test(s)).filter(isExpectedError(cwd)).map(lines => lines.join("\n")).join("\n"); +} +function isExpectedError(cwd: string) { + return (error: string[]) => { + if (error.length === 0) { + return true; + } + const match = error[0].match(/(.+\.ts)\((\d+),\d+\): error TS/); + if (!match) { + return true; + } + const errlines = fs.readFileSync(path.join(cwd, match[1]), { encoding: "utf8" }).split("\n"); + const index = parseInt(match[2]); + const errline = index < errlines.length ? errlines[index] : ""; + const prevline = index - 1 < errlines.length && index > 0 ? errlines[index - 1] : ""; + if (errline.indexOf("$ExpectError") > -1 || prevline.indexOf("$ExpectError") > -1) { + return false; + } + return true; + } +} +function breaks(xs: T[], isStart: (T: any) => boolean): T[][] { + const result = []; + let group: T[] = []; + for (const x of xs) { + if (isStart(x)) { + if (group.length) { + result.push(group); + } + group = [x]; + } + else { + group.push(x); + } + } + if (group.length) { + result.push(group); + } + return result; } From 0d5800a17bc0ed88094b5dc3e8190128f4f71446 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Fri, 10 Nov 2017 09:37:06 -0800 Subject: [PATCH 2/3] Address PR comments --- src/harness/externalCompileRunner.ts | 62 +++++++++++++++++----------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/src/harness/externalCompileRunner.ts b/src/harness/externalCompileRunner.ts index ec8acf4872e..5730c5c3c83 100644 --- a/src/harness/externalCompileRunner.ts +++ b/src/harness/externalCompileRunner.ts @@ -2,9 +2,16 @@ /// const fs = require("fs"); const path = require("path"); + +interface ExecResult { + stdout: Buffer; + stderr: Buffer; + status: number; +} + abstract class ExternalCompileRunnerBase extends RunnerBase { abstract testDir: string; - abstract report(result: any, cwd: string): string; + abstract report(result: ExecResult, cwd: string): string; enumerateTestFiles() { return Harness.IO.getDirectories(this.testDir); } @@ -24,8 +31,6 @@ abstract class ExternalCompileRunnerBase extends RunnerBase { private runTest(directoryName: string) { describe(directoryName, () => { const cp = require("child_process"); - const path = require("path"); - const fs = require("fs"); it("should build successfully", () => { const cwd = path.join(__dirname, "../../", this.testDir, directoryName); @@ -51,7 +56,7 @@ class UserCodeRunner extends ExternalCompileRunnerBase { kind(): TestRunnerKind { return "user"; } - report(result: any) { + report(result: ExecResult) { // tslint:disable-next-line:no-null-keyword return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} Standard output: @@ -69,9 +74,9 @@ class DefinitelyTypedRunner extends ExternalCompileRunnerBase { kind(): TestRunnerKind { return "dt"; } - report(result: any, cwd: string) { - const stdout = filterExpectedErrors(result.stdout.toString(), cwd) - const stderr = result.stderr.toString() + report(result: ExecResult, cwd: string) { + const stdout = removeExpectedErrors(result.stdout.toString(), cwd); + const stderr = result.stderr.toString(); // tslint:disable-next-line:no-null-keyword return !stdout.length && !stderr.length ? null : `Exit Code: ${result.status} Standard output: @@ -83,29 +88,40 @@ ${stderr.replace(/\r\n/g, "\n")}`; } } -function filterExpectedErrors(errors: string, cwd: string): string { - return breaks(errors.split("\n"), s => /^\w+/.test(s)).filter(isExpectedError(cwd)).map(lines => lines.join("\n")).join("\n"); +function removeExpectedErrors(errors: string, cwd: string): string { + return ts.flatten(splitBy(errors.split("\n"), s => /^\S+/.test(s)).filter(isUnexpectedError(cwd))).join("\n"); } -function isExpectedError(cwd: string) { +/** + * Returns true if the line that caused the error contains '$ExpectError', + * or if the line before that one contains '$ExpectError'. + * '$ExpectError' is a marker used in Definitely Typed tests, + * meaning that the error should not contribute toward our error baslines. + */ +function isUnexpectedError(cwd: string) { return (error: string[]) => { - if (error.length === 0) { - return true; - } + ts.Debug.assertGreaterThanOrEqual(error.length, 1); const match = error[0].match(/(.+\.ts)\((\d+),\d+\): error TS/); if (!match) { return true; } - const errlines = fs.readFileSync(path.join(cwd, match[1]), { encoding: "utf8" }).split("\n"); - const index = parseInt(match[2]); - const errline = index < errlines.length ? errlines[index] : ""; - const prevline = index - 1 < errlines.length && index > 0 ? errlines[index - 1] : ""; - if (errline.indexOf("$ExpectError") > -1 || prevline.indexOf("$ExpectError") > -1) { - return false; - } - return true; - } + const [, errorFile, lineNumberString] = match; + const lines = fs.readFileSync(path.join(cwd, errorFile), { encoding: "utf8" }).split("\n"); + const lineNumber = parseInt(lineNumberString); + ts.Debug.assertGreaterThanOrEqual(lineNumber, 0); + ts.Debug.assertLessThan(lineNumber, lines.length); + const previousLine = lineNumber - 1 > 0 ? lines[lineNumber - 1] : ""; + return lines[lineNumber].indexOf("$ExpectError") === -1 && previousLine.indexOf("$ExpectError") === -1; + }; } -function breaks(xs: T[], isStart: (T: any) => boolean): T[][] { +/** + * Split an array into multiple arrays whenever `isStart` returns true. + * @example + * splitBy([1,2,3,4,5,6], isOdd) + * ==> [[1, 2], [3, 4], [5, 6]] + * where + * const isOdd = n => !!(n % 2) + */ +function splitBy(xs: T[], isStart: (x: T) => boolean): T[][] { const result = []; let group: T[] = []; for (const x of xs) { From 5fff71742b019be600b5bfe4c60e03afc73fe594 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Fri, 10 Nov 2017 10:11:44 -0800 Subject: [PATCH 3/3] Use ts.stringContains instead of String.indexOf --- src/harness/externalCompileRunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/harness/externalCompileRunner.ts b/src/harness/externalCompileRunner.ts index 5730c5c3c83..33803752f94 100644 --- a/src/harness/externalCompileRunner.ts +++ b/src/harness/externalCompileRunner.ts @@ -110,7 +110,7 @@ function isUnexpectedError(cwd: string) { ts.Debug.assertGreaterThanOrEqual(lineNumber, 0); ts.Debug.assertLessThan(lineNumber, lines.length); const previousLine = lineNumber - 1 > 0 ? lines[lineNumber - 1] : ""; - return lines[lineNumber].indexOf("$ExpectError") === -1 && previousLine.indexOf("$ExpectError") === -1; + return !ts.stringContains(lines[lineNumber], "$ExpectError") && !ts.stringContains(previousLine, "$ExpectError"); }; } /**