From e6c38bf67b451794471245b4a9f4909d02f66320 Mon Sep 17 00:00:00 2001
From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com>
Date: Mon, 6 Nov 2017 15:16:33 -0800
Subject: [PATCH] Add DefinitelyTyped test runner

Assumes that ../DefinitelyTyped holds the DefinitelyTyped repo.
---
 src/harness/definitelyRunner.ts | 51 +++++++++++++++++++++++++++++++++
 src/harness/runner.ts           |  6 ++++
 src/harness/runnerbase.ts       |  2 +-
 src/harness/tsconfig.json       |  1 +
 src/harness/userRunner.ts       |  2 +-
 5 files changed, 60 insertions(+), 2 deletions(-)
 create mode 100644 src/harness/definitelyRunner.ts

diff --git a/src/harness/definitelyRunner.ts b/src/harness/definitelyRunner.ts
new file mode 100644
index 00000000000..afd39b72424
--- /dev/null
+++ b/src/harness/definitelyRunner.ts
@@ -0,0 +1,51 @@
+/// <reference path="harness.ts"/>
+/// <reference path="runnerbase.ts" />
+class DefinitelyTypedRunner extends RunnerBase {
+    private static readonly testDir = "../DefinitelyTyped/types/";
+    public enumerateTestFiles() {
+        return Harness.IO.getDirectories(DefinitelyTypedRunner.testDir).map(dir => DefinitelyTypedRunner.testDir + dir);
+    }
+
+    public kind(): TestRunnerKind {
+        return "definitely";
+    }
+
+    /** Setup the runner's tests so that they are ready to be executed by the harness
+     *  The first test should be a describe/it block that sets up the harness's compiler instance appropriately
+     */
+    public initializeTests(): void {
+        // Read in and evaluate the test list
+        const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
+
+        describe(`${this.kind()} code samples`, () => {
+            for (const test of testList) {
+                this.runTest(test);
+            }
+        });
+    }
+
+    private runTest(directoryName: string) {
+        describe(directoryName, () => {
+            const cp = require("child_process");
+            const path = require("path");
+
+            it("should build successfully", () => {
+                const cwd = path.join(__dirname, "../../", directoryName);
+                const timeout = 600000; // 600s = 10 minutes
+                const stdio = isWorker ? "pipe" : "inherit";
+                const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio });
+                if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`);
+                Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => {
+                    const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js"), "--lib dom,es6", "--strict"], { cwd, timeout, shell: true });
+                    return `Exit Code: ${result.status}
+Standard output:
+${result.stdout.toString().replace(/\r\n/g, "\n")}
+
+
+Standard error:
+${result.stderr.toString().replace(/\r\n/g, "\n")}`;
+                });
+            });
+        });
+    }
+}
diff --git a/src/harness/runner.ts b/src/harness/runner.ts
index 70954e9e853..fb66e74b958 100644
--- a/src/harness/runner.ts
+++ b/src/harness/runner.ts
@@ -19,6 +19,7 @@
 /// <reference path="projectsRunner.ts" />
 /// <reference path="rwcRunner.ts" />
 /// <reference path="userRunner.ts" />
+/// <reference path="definitelyRunner.ts" />
 /// <reference path="harness.ts" />
 /// <reference path="./parallel/shared.ts" />
 
@@ -62,6 +63,8 @@ function createRunner(kind: TestRunnerKind): RunnerBase {
             return new Test262BaselineRunner();
         case "user":
             return new UserCodeRunner();
+        case "definitely":
+            return new DefinitelyTypedRunner();
     }
     ts.Debug.fail(`Unknown runner kind ${kind}`);
 }
@@ -183,6 +186,9 @@ function handleTestConfig() {
                     case "user":
                         runners.push(new UserCodeRunner());
                         break;
+                    case "definitely":
+                        runners.push(new DefinitelyTypedRunner());
+                        break;
                 }
             }
         }
diff --git a/src/harness/runnerbase.ts b/src/harness/runnerbase.ts
index 2fef2264b73..42e625a897d 100644
--- a/src/harness/runnerbase.ts
+++ b/src/harness/runnerbase.ts
@@ -1,7 +1,7 @@
 /// <reference path="harness.ts" />
 
 
-type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user";
+type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user" | "definitely";
 type CompilerTestKind = "conformance" | "compiler";
 type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
 
diff --git a/src/harness/tsconfig.json b/src/harness/tsconfig.json
index 6e61b7690bc..96f1999e9e8 100644
--- a/src/harness/tsconfig.json
+++ b/src/harness/tsconfig.json
@@ -93,6 +93,7 @@
         "loggedIO.ts",
         "rwcRunner.ts",
         "userRunner.ts",
+        "definitelyRunner.ts",
         "test262Runner.ts",
         "./parallel/shared.ts",
         "./parallel/host.ts",
diff --git a/src/harness/userRunner.ts b/src/harness/userRunner.ts
index 9be652aebf8..61a46d7e84f 100644
--- a/src/harness/userRunner.ts
+++ b/src/harness/userRunner.ts
@@ -36,7 +36,7 @@ class UserCodeRunner extends RunnerBase {
                 const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio });
                 if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`);
                 Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => {
-                    const result = cp.spawnSync(`node`, ["../../../../built/local/tsc.js"], { cwd, timeout, shell: true });
+                    const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true });
                     return `Exit Code: ${result.status}
 Standard output:
 ${result.stdout.toString().replace(/\r\n/g, "\n")}