mirror of
https://github.com/microsoft/TypeScript.git
synced 2025-11-18 17:21:48 +00:00
Parallel linting (#10313)
* A perilous thing, a parallel lint * Use work queue rather than scheduling work * Dont read files for lint on main thread * Fix style
This commit is contained in:
Wesley Wigham
GitHub
+60
-38
@@ -34,7 +34,6 @@ import through2 = require("through2");
|
||||
import merge2 = require("merge2");
|
||||
import intoStream = require("into-stream");
|
||||
import * as os from "os";
|
||||
import Linter = require("tslint");
|
||||
import fold = require("travis-fold");
|
||||
const gulp = helpMaker(originalGulp);
|
||||
const mochaParallel = require("./scripts/mocha-parallel.js");
|
||||
@@ -929,26 +928,6 @@ gulp.task("build-rules", "Compiles tslint rules to js", () => {
|
||||
.pipe(gulp.dest(dest));
|
||||
});
|
||||
|
||||
function getLinterOptions() {
|
||||
return {
|
||||
configuration: require("./tslint.json"),
|
||||
formatter: "prose",
|
||||
formattersDirectory: undefined,
|
||||
rulesDirectory: "built/local/tslint"
|
||||
};
|
||||
}
|
||||
|
||||
function lintFileContents(options, path, contents) {
|
||||
const ll = new Linter(path, contents, options);
|
||||
console.log("Linting '" + path + "'.");
|
||||
return ll.lint();
|
||||
}
|
||||
|
||||
function lintFile(options, path) {
|
||||
const contents = fs.readFileSync(path, "utf8");
|
||||
return lintFileContents(options, path, contents);
|
||||
}
|
||||
|
||||
const lintTargets = [
|
||||
"Gulpfile.ts",
|
||||
"src/compiler/**/*.ts",
|
||||
@@ -960,29 +939,72 @@ const lintTargets = [
|
||||
"tests/*.ts", "tests/webhost/*.ts" // Note: does *not* descend recursively
|
||||
];
|
||||
|
||||
function sendNextFile(files: {path: string}[], child: cp.ChildProcess, callback: (failures: number) => void, failures: number) {
|
||||
const file = files.pop();
|
||||
if (file) {
|
||||
console.log(`Linting '${file.path}'.`);
|
||||
child.send({ kind: "file", name: file.path });
|
||||
}
|
||||
else {
|
||||
child.send({ kind: "close" });
|
||||
callback(failures);
|
||||
}
|
||||
}
|
||||
|
||||
function spawnLintWorker(files: {path: string}[], callback: (failures: number) => void) {
|
||||
const child = cp.fork("./scripts/parallel-lint");
|
||||
let failures = 0;
|
||||
child.on("message", function(data) {
|
||||
switch (data.kind) {
|
||||
case "result":
|
||||
if (data.failures > 0) {
|
||||
failures += data.failures;
|
||||
console.log(data.output);
|
||||
}
|
||||
sendNextFile(files, child, callback, failures);
|
||||
break;
|
||||
case "error":
|
||||
console.error(data.error);
|
||||
failures++;
|
||||
sendNextFile(files, child, callback, failures);
|
||||
break;
|
||||
}
|
||||
});
|
||||
sendNextFile(files, child, callback, failures);
|
||||
}
|
||||
|
||||
gulp.task("lint", "Runs tslint on the compiler sources. Optional arguments are: --f[iles]=regex", ["build-rules"], () => {
|
||||
const fileMatcher = RegExp(cmdLineOptions["files"]);
|
||||
const lintOptions = getLinterOptions();
|
||||
let failed = 0;
|
||||
if (fold.isTravis()) console.log(fold.start("lint"));
|
||||
return gulp.src(lintTargets)
|
||||
.pipe(insert.transform((contents, file) => {
|
||||
if (!fileMatcher.test(file.path)) return contents;
|
||||
const result = lintFile(lintOptions, file.path);
|
||||
if (result.failureCount > 0) {
|
||||
console.log(result.output);
|
||||
failed += result.failureCount;
|
||||
|
||||
const files: {stat: fs.Stats, path: string}[] = [];
|
||||
return gulp.src(lintTargets, { read: false })
|
||||
.pipe(through2.obj((chunk, enc, cb) => {
|
||||
files.push(chunk);
|
||||
cb();
|
||||
}, (cb) => {
|
||||
files.filter(file => fileMatcher.test(file.path)).sort((filea, fileb) => filea.stat.size - fileb.stat.size);
|
||||
const workerCount = (process.env.workerCount && +process.env.workerCount) || os.cpus().length;
|
||||
for (let i = 0; i < workerCount; i++) {
|
||||
spawnLintWorker(files, finished);
|
||||
}
|
||||
return contents; // TODO (weswig): Automatically apply fixes? :3
|
||||
}))
|
||||
.on("end", () => {
|
||||
if (fold.isTravis()) console.log(fold.end("lint"));
|
||||
if (failed > 0) {
|
||||
console.error("Linter errors.");
|
||||
process.exit(1);
|
||||
|
||||
let completed = 0;
|
||||
let failures = 0;
|
||||
function finished(fails) {
|
||||
completed++;
|
||||
failures += fails;
|
||||
if (completed === workerCount) {
|
||||
if (fold.isTravis()) console.log(fold.end("lint"));
|
||||
if (failures > 0) {
|
||||
throw new Error(`Linter errors: ${failures}`);
|
||||
}
|
||||
else {
|
||||
cb();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}));
|
||||
});
|
||||
|
||||
|
||||
|
||||
+57
-85
@@ -4,7 +4,6 @@ var fs = require("fs");
|
||||
var os = require("os");
|
||||
var path = require("path");
|
||||
var child_process = require("child_process");
|
||||
var Linter = require("tslint");
|
||||
var fold = require("travis-fold");
|
||||
var runTestsInParallel = require("./scripts/mocha-parallel").runTestsInParallel;
|
||||
|
||||
@@ -1054,36 +1053,6 @@ task("build-rules-end", [] , function() {
|
||||
if (fold.isTravis()) console.log(fold.end("build-rules"));
|
||||
});
|
||||
|
||||
function getLinterOptions() {
|
||||
return {
|
||||
configuration: require("./tslint.json"),
|
||||
formatter: "prose",
|
||||
formattersDirectory: undefined,
|
||||
rulesDirectory: "built/local/tslint"
|
||||
};
|
||||
}
|
||||
|
||||
function lintFileContents(options, path, contents) {
|
||||
var ll = new Linter(path, contents, options);
|
||||
console.log("Linting '" + path + "'.");
|
||||
return ll.lint();
|
||||
}
|
||||
|
||||
function lintFile(options, path) {
|
||||
var contents = fs.readFileSync(path, "utf8");
|
||||
return lintFileContents(options, path, contents);
|
||||
}
|
||||
|
||||
function lintFileAsync(options, path, cb) {
|
||||
fs.readFile(path, "utf8", function(err, contents) {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
var result = lintFileContents(options, path, contents);
|
||||
cb(undefined, result);
|
||||
});
|
||||
}
|
||||
|
||||
var lintTargets = compilerSources
|
||||
.concat(harnessSources)
|
||||
// Other harness sources
|
||||
@@ -1094,75 +1063,78 @@ var lintTargets = compilerSources
|
||||
.concat(["Gulpfile.ts"])
|
||||
.concat([nodeServerInFile, perftscPath, "tests/perfsys.ts", webhostPath]);
|
||||
|
||||
function sendNextFile(files, child, callback, failures) {
|
||||
var file = files.pop();
|
||||
if (file) {
|
||||
console.log("Linting '" + file + "'.");
|
||||
child.send({kind: "file", name: file});
|
||||
}
|
||||
else {
|
||||
child.send({kind: "close"});
|
||||
callback(failures);
|
||||
}
|
||||
}
|
||||
|
||||
function spawnLintWorker(files, callback) {
|
||||
var child = child_process.fork("./scripts/parallel-lint");
|
||||
var failures = 0;
|
||||
child.on("message", function(data) {
|
||||
switch (data.kind) {
|
||||
case "result":
|
||||
if (data.failures > 0) {
|
||||
failures += data.failures;
|
||||
console.log(data.output);
|
||||
}
|
||||
sendNextFile(files, child, callback, failures);
|
||||
break;
|
||||
case "error":
|
||||
console.error(data.error);
|
||||
failures++;
|
||||
sendNextFile(files, child, callback, failures);
|
||||
break;
|
||||
}
|
||||
});
|
||||
sendNextFile(files, child, callback, failures);
|
||||
}
|
||||
|
||||
desc("Runs tslint on the compiler sources. Optional arguments are: f[iles]=regex");
|
||||
task("lint", ["build-rules"], function() {
|
||||
if (fold.isTravis()) console.log(fold.start("lint"));
|
||||
var startTime = mark();
|
||||
var lintOptions = getLinterOptions();
|
||||
var failed = 0;
|
||||
var fileMatcher = RegExp(process.env.f || process.env.file || process.env.files || "");
|
||||
var done = {};
|
||||
for (var i in lintTargets) {
|
||||
var target = lintTargets[i];
|
||||
if (!done[target] && fileMatcher.test(target)) {
|
||||
var result = lintFile(lintOptions, target);
|
||||
if (result.failureCount > 0) {
|
||||
console.log(result.output);
|
||||
failed += result.failureCount;
|
||||
}
|
||||
done[target] = true;
|
||||
done[target] = fs.statSync(target).size;
|
||||
}
|
||||
}
|
||||
measure(startTime);
|
||||
if (fold.isTravis()) console.log(fold.end("lint"));
|
||||
if (failed > 0) {
|
||||
fail('Linter errors.', failed);
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* This is required because file watches on Windows get fires _twice_
|
||||
* when a file changes on some node/windows version configuations
|
||||
* (node v4 and win 10, for example). By not running a lint for a file
|
||||
* which already has a pending lint, we avoid duplicating our work.
|
||||
* (And avoid printing duplicate results!)
|
||||
*/
|
||||
var lintSemaphores = {};
|
||||
var workerCount = (process.env.workerCount && +process.env.workerCount) || os.cpus().length;
|
||||
|
||||
function lintWatchFile(filename) {
|
||||
fs.watch(filename, {persistent: true}, function(event) {
|
||||
if (event !== "change") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!lintSemaphores[filename]) {
|
||||
lintSemaphores[filename] = true;
|
||||
lintFileAsync(getLinterOptions(), filename, function(err, result) {
|
||||
delete lintSemaphores[filename];
|
||||
if (err) {
|
||||
console.log(err);
|
||||
return;
|
||||
}
|
||||
if (result.failureCount > 0) {
|
||||
console.log("***Lint failure***");
|
||||
for (var i = 0; i < result.failures.length; i++) {
|
||||
var failure = result.failures[i];
|
||||
var start = failure.startPosition.lineAndCharacter;
|
||||
var end = failure.endPosition.lineAndCharacter;
|
||||
console.log("warning " + filename + " (" + (start.line + 1) + "," + (start.character + 1) + "," + (end.line + 1) + "," + (end.character + 1) + "): " + failure.failure);
|
||||
}
|
||||
console.log("*** Total " + result.failureCount + " failures.");
|
||||
}
|
||||
});
|
||||
}
|
||||
var names = Object.keys(done).sort(function(namea, nameb) {
|
||||
return done[namea] - done[nameb];
|
||||
});
|
||||
}
|
||||
|
||||
desc("Watches files for changes to rerun a lint pass");
|
||||
task("lint-server", ["build-rules"], function() {
|
||||
console.log("Watching ./src for changes to linted files");
|
||||
for (var i = 0; i < lintTargets.length; i++) {
|
||||
lintWatchFile(lintTargets[i]);
|
||||
for (var i = 0; i < workerCount; i++) {
|
||||
spawnLintWorker(names, finished);
|
||||
}
|
||||
});
|
||||
|
||||
var completed = 0;
|
||||
var failures = 0;
|
||||
function finished(fails) {
|
||||
completed++;
|
||||
failures += fails;
|
||||
if (completed === workerCount) {
|
||||
measure(startTime);
|
||||
if (fold.isTravis()) console.log(fold.end("lint"));
|
||||
if (failures > 0) {
|
||||
fail('Linter errors.', failed);
|
||||
}
|
||||
else {
|
||||
complete();
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {async: true});
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
var Linter = require("tslint");
|
||||
var fs = require("fs");
|
||||
|
||||
function getLinterOptions() {
|
||||
return {
|
||||
configuration: require("../tslint.json"),
|
||||
formatter: "prose",
|
||||
formattersDirectory: undefined,
|
||||
rulesDirectory: "built/local/tslint"
|
||||
};
|
||||
}
|
||||
|
||||
function lintFileContents(options, path, contents) {
|
||||
var ll = new Linter(path, contents, options);
|
||||
return ll.lint();
|
||||
}
|
||||
|
||||
function lintFileAsync(options, path, cb) {
|
||||
fs.readFile(path, "utf8", function (err, contents) {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
var result = lintFileContents(options, path, contents);
|
||||
cb(undefined, result);
|
||||
});
|
||||
}
|
||||
|
||||
process.on("message", function (data) {
|
||||
switch (data.kind) {
|
||||
case "file":
|
||||
var target = data.name;
|
||||
var lintOptions = getLinterOptions();
|
||||
lintFileAsync(lintOptions, target, function (err, result) {
|
||||
if (err) {
|
||||
process.send({ kind: "error", error: err.toString() });
|
||||
return;
|
||||
}
|
||||
process.send({ kind: "result", failures: result.failureCount, output: result.output });
|
||||
});
|
||||
break;
|
||||
case "close":
|
||||
process.exit(0);
|
||||
break;
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user