From a5fa75a0cd095702d41e259cd2eb73d2c777b162 Mon Sep 17 00:00:00 2001
From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com>
Date: Thu, 9 Nov 2017 14:21:08 -0800
Subject: [PATCH 1/3] DefinitelyTypedRunner skips ExpectErrors
If all errors were ExpectErrors, then it does not fail.
---
src/harness/externalCompileRunner.ts | 91 +++++++++++++++++++++++-----
1 file changed, 75 insertions(+), 16 deletions(-)
diff --git a/src/harness/externalCompileRunner.ts b/src/harness/externalCompileRunner.ts
index 9b8fc8c4fc5..ec8acf4872e 100644
--- a/src/harness/externalCompileRunner.ts
+++ b/src/harness/externalCompileRunner.ts
@@ -1,14 +1,17 @@
///
///
+const fs = require("fs");
+const path = require("path");
abstract class ExternalCompileRunnerBase extends RunnerBase {
abstract testDir: string;
- public enumerateTestFiles() {
+ abstract report(result: any, cwd: string): string;
+ enumerateTestFiles() {
return Harness.IO.getDirectories(this.testDir);
}
/** Setup the runner's tests so that they are ready to be executed by the harness
* The first test should be a describe/it block that sets up the harness's compiler instance appropriately
*/
- public initializeTests(): void {
+ initializeTests(): void {
// Read in and evaluate the test list
const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
@@ -36,15 +39,7 @@ abstract class ExternalCompileRunnerBase extends RunnerBase {
if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`);
}
Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => {
- const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true });
- // tslint:disable-next-line:no-null-keyword
- return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
-Standard output:
-${result.stdout.toString().replace(/\r\n/g, "\n")}
-
-
-Standard error:
-${result.stderr.toString().replace(/\r\n/g, "\n")}`;
+ return this.report(cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true }), cwd);
});
});
});
@@ -52,16 +47,80 @@ ${result.stderr.toString().replace(/\r\n/g, "\n")}`;
}
class UserCodeRunner extends ExternalCompileRunnerBase {
- public readonly testDir = "tests/cases/user/";
- public kind(): TestRunnerKind {
+ readonly testDir = "tests/cases/user/";
+ kind(): TestRunnerKind {
return "user";
}
+ report(result: any) {
+ // tslint:disable-next-line:no-null-keyword
+ return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
+Standard output:
+${result.stdout.toString().replace(/\r\n/g, "\n")}
+
+
+Standard error:
+${result.stderr.toString().replace(/\r\n/g, "\n")}`;
+ }
}
class DefinitelyTypedRunner extends ExternalCompileRunnerBase {
- public readonly testDir = "../DefinitelyTyped/types/";
- public workingDirectory = this.testDir;
- public kind(): TestRunnerKind {
+ readonly testDir = "../DefinitelyTyped/types/";
+ workingDirectory = this.testDir;
+ kind(): TestRunnerKind {
return "dt";
}
+ report(result: any, cwd: string) {
+ const stdout = filterExpectedErrors(result.stdout.toString(), cwd)
+ const stderr = result.stderr.toString()
+ // tslint:disable-next-line:no-null-keyword
+ return !stdout.length && !stderr.length ? null : `Exit Code: ${result.status}
+Standard output:
+${stdout.replace(/\r\n/g, "\n")}
+
+
+Standard error:
+${stderr.replace(/\r\n/g, "\n")}`;
+ }
+}
+
+function filterExpectedErrors(errors: string, cwd: string): string {
+ return breaks(errors.split("\n"), s => /^\w+/.test(s)).filter(isExpectedError(cwd)).map(lines => lines.join("\n")).join("\n");
+}
+function isExpectedError(cwd: string) {
+ return (error: string[]) => {
+ if (error.length === 0) {
+ return true;
+ }
+ const match = error[0].match(/(.+\.ts)\((\d+),\d+\): error TS/);
+ if (!match) {
+ return true;
+ }
+ const errlines = fs.readFileSync(path.join(cwd, match[1]), { encoding: "utf8" }).split("\n");
+ const index = parseInt(match[2]);
+ const errline = index < errlines.length ? errlines[index] : "";
+ const prevline = index - 1 < errlines.length && index > 0 ? errlines[index - 1] : "";
+ if (errline.indexOf("$ExpectError") > -1 || prevline.indexOf("$ExpectError") > -1) {
+ return false;
+ }
+ return true;
+ }
+}
+function breaks(xs: T[], isStart: (T: any) => boolean): T[][] {
+ const result = [];
+ let group: T[] = [];
+ for (const x of xs) {
+ if (isStart(x)) {
+ if (group.length) {
+ result.push(group);
+ }
+ group = [x];
+ }
+ else {
+ group.push(x);
+ }
+ }
+ if (group.length) {
+ result.push(group);
+ }
+ return result;
}
From 0d5800a17bc0ed88094b5dc3e8190128f4f71446 Mon Sep 17 00:00:00 2001
From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com>
Date: Fri, 10 Nov 2017 09:37:06 -0800
Subject: [PATCH 2/3] Address PR comments
---
src/harness/externalCompileRunner.ts | 62 +++++++++++++++++-----------
1 file changed, 39 insertions(+), 23 deletions(-)
diff --git a/src/harness/externalCompileRunner.ts b/src/harness/externalCompileRunner.ts
index ec8acf4872e..5730c5c3c83 100644
--- a/src/harness/externalCompileRunner.ts
+++ b/src/harness/externalCompileRunner.ts
@@ -2,9 +2,16 @@
///
const fs = require("fs");
const path = require("path");
+
+interface ExecResult {
+ stdout: Buffer;
+ stderr: Buffer;
+ status: number;
+}
+
abstract class ExternalCompileRunnerBase extends RunnerBase {
abstract testDir: string;
- abstract report(result: any, cwd: string): string;
+ abstract report(result: ExecResult, cwd: string): string;
enumerateTestFiles() {
return Harness.IO.getDirectories(this.testDir);
}
@@ -24,8 +31,6 @@ abstract class ExternalCompileRunnerBase extends RunnerBase {
private runTest(directoryName: string) {
describe(directoryName, () => {
const cp = require("child_process");
- const path = require("path");
- const fs = require("fs");
it("should build successfully", () => {
const cwd = path.join(__dirname, "../../", this.testDir, directoryName);
@@ -51,7 +56,7 @@ class UserCodeRunner extends ExternalCompileRunnerBase {
kind(): TestRunnerKind {
return "user";
}
- report(result: any) {
+ report(result: ExecResult) {
// tslint:disable-next-line:no-null-keyword
return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
Standard output:
@@ -69,9 +74,9 @@ class DefinitelyTypedRunner extends ExternalCompileRunnerBase {
kind(): TestRunnerKind {
return "dt";
}
- report(result: any, cwd: string) {
- const stdout = filterExpectedErrors(result.stdout.toString(), cwd)
- const stderr = result.stderr.toString()
+ report(result: ExecResult, cwd: string) {
+ const stdout = removeExpectedErrors(result.stdout.toString(), cwd);
+ const stderr = result.stderr.toString();
// tslint:disable-next-line:no-null-keyword
return !stdout.length && !stderr.length ? null : `Exit Code: ${result.status}
Standard output:
@@ -83,29 +88,40 @@ ${stderr.replace(/\r\n/g, "\n")}`;
}
}
-function filterExpectedErrors(errors: string, cwd: string): string {
- return breaks(errors.split("\n"), s => /^\w+/.test(s)).filter(isExpectedError(cwd)).map(lines => lines.join("\n")).join("\n");
+function removeExpectedErrors(errors: string, cwd: string): string {
+ return ts.flatten(splitBy(errors.split("\n"), s => /^\S+/.test(s)).filter(isUnexpectedError(cwd))).join("\n");
}
-function isExpectedError(cwd: string) {
+/**
+ * Returns true if the line that caused the error contains '$ExpectError',
+ * or if the line before that one contains '$ExpectError'.
+ * '$ExpectError' is a marker used in Definitely Typed tests,
+ * meaning that the error should not contribute toward our error baslines.
+ */
+function isUnexpectedError(cwd: string) {
return (error: string[]) => {
- if (error.length === 0) {
- return true;
- }
+ ts.Debug.assertGreaterThanOrEqual(error.length, 1);
const match = error[0].match(/(.+\.ts)\((\d+),\d+\): error TS/);
if (!match) {
return true;
}
- const errlines = fs.readFileSync(path.join(cwd, match[1]), { encoding: "utf8" }).split("\n");
- const index = parseInt(match[2]);
- const errline = index < errlines.length ? errlines[index] : "";
- const prevline = index - 1 < errlines.length && index > 0 ? errlines[index - 1] : "";
- if (errline.indexOf("$ExpectError") > -1 || prevline.indexOf("$ExpectError") > -1) {
- return false;
- }
- return true;
- }
+ const [, errorFile, lineNumberString] = match;
+ const lines = fs.readFileSync(path.join(cwd, errorFile), { encoding: "utf8" }).split("\n");
+ const lineNumber = parseInt(lineNumberString);
+ ts.Debug.assertGreaterThanOrEqual(lineNumber, 0);
+ ts.Debug.assertLessThan(lineNumber, lines.length);
+ const previousLine = lineNumber - 1 > 0 ? lines[lineNumber - 1] : "";
+ return lines[lineNumber].indexOf("$ExpectError") === -1 && previousLine.indexOf("$ExpectError") === -1;
+ };
}
-function breaks(xs: T[], isStart: (T: any) => boolean): T[][] {
+/**
+ * Split an array into multiple arrays whenever `isStart` returns true.
+ * @example
+ * splitBy([1,2,3,4,5,6], isOdd)
+ * ==> [[1, 2], [3, 4], [5, 6]]
+ * where
+ * const isOdd = n => !!(n % 2)
+ */
+function splitBy(xs: T[], isStart: (x: T) => boolean): T[][] {
const result = [];
let group: T[] = [];
for (const x of xs) {
From 5fff71742b019be600b5bfe4c60e03afc73fe594 Mon Sep 17 00:00:00 2001
From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com>
Date: Fri, 10 Nov 2017 10:11:44 -0800
Subject: [PATCH 3/3] Use ts.stringContains instead of String.indexOf
---
src/harness/externalCompileRunner.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/harness/externalCompileRunner.ts b/src/harness/externalCompileRunner.ts
index 5730c5c3c83..33803752f94 100644
--- a/src/harness/externalCompileRunner.ts
+++ b/src/harness/externalCompileRunner.ts
@@ -110,7 +110,7 @@ function isUnexpectedError(cwd: string) {
ts.Debug.assertGreaterThanOrEqual(lineNumber, 0);
ts.Debug.assertLessThan(lineNumber, lines.length);
const previousLine = lineNumber - 1 > 0 ? lines[lineNumber - 1] : "";
- return lines[lineNumber].indexOf("$ExpectError") === -1 && previousLine.indexOf("$ExpectError") === -1;
+ return !ts.stringContains(lines[lineNumber], "$ExpectError") && !ts.stringContains(previousLine, "$ExpectError");
};
}
/**