Salsa: JS support for discovering and acquiring d.ts files

(Mostly isolating VS host changes from PR#6448)

Jakefile.js

@@ -920,6 +920,7 @@ var servicesLintTargets = [
     "patternMatcher.ts",
     "services.ts",
     "shims.ts",
+    "jsTyping.ts"
 ].map(function (s) {
     return path.join(servicesDirectory, s);
 });

src/compiler/commandLineParser.ts

@@ -511,6 +511,7 @@ namespace ts {
         return {
             options,
             fileNames: getFileNames(),
+            typingOptions: getTypingOptions(),
             errors
         };
 
@@ -575,6 +576,32 @@ namespace ts {
             }
             return fileNames;
         }
+
+        function getTypingOptions(): TypingOptions {
+            const options: TypingOptions = getBaseFileName(configFileName) === "jsconfig.json"
+                ? { enableAutoDiscovery: true, include: [], exclude: [] }
+                : { enableAutoDiscovery: false, include: [], exclude: [] };
+            const jsonTypingOptions = json["typingOptions"];
+            if (jsonTypingOptions) {
+                for (const id in jsonTypingOptions) {
+                    if (id === "enableAutoDiscovery") {
+                        if (typeof jsonTypingOptions[id] === "boolean") {
+                            options.enableAutoDiscovery = jsonTypingOptions[id];
+                        }
+                    }
+                    else if (id === "include") {
+                        options.include = isArray(jsonTypingOptions[id]) ? <string[]>jsonTypingOptions[id] : [];
+                    }
+                    else if (id === "exclude") {
+                        options.exclude = isArray(jsonTypingOptions[id]) ? <string[]>jsonTypingOptions[id] : [];
+                    }
+                    else {
+                        errors.push(createCompilerDiagnostic(Diagnostics.Unknown_typing_option_0, id));
+                    }
+                }
+            }
+            return options;
+        }
     }
 
     export function convertCompilerOptionsFromJson(jsonOptions: any, basePath: string, configFileName?: string): { options: CompilerOptions, errors: Diagnostic[] } {

src/compiler/core.ts

@@ -769,6 +769,32 @@ namespace ts {
         return pathLen > extLen && path.substr(pathLen - extLen, extLen) === extension;
     }
 
+    export function ensureScriptKind(fileName: string, scriptKind?: ScriptKind): ScriptKind {
+        // Using scriptKind as a condition handles both:
+        // - 'scriptKind' is unspecified and thus it is `undefined`
+        // - 'scriptKind' is set and it is `Unknown` (0)
+        // If the 'scriptKind' is 'undefined' or 'Unknown' then we attempt
+        // to get the ScriptKind from the file name. If it cannot be resolved
+        // from the file name then the default 'TS' script kind is returned.
+        return (scriptKind || getScriptKindFromFileName(fileName)) || ScriptKind.TS;
+    }
+
+    export function getScriptKindFromFileName(fileName: string): ScriptKind {
+        const ext = fileName.substr(fileName.lastIndexOf("."));
+        switch (ext.toLowerCase()) {
+            case ".js":
+                return ScriptKind.JS;
+            case ".jsx":
+                return ScriptKind.JSX;
+            case ".ts":
+                return ScriptKind.TS;
+            case ".tsx":
+                return ScriptKind.TSX;
+            default:
+                return ScriptKind.Unknown;
+        }
+    }
+
     /**
      *  List of supported extensions in order of file resolution precedence.
      */

src/compiler/diagnosticMessages.json

@@ -2654,5 +2654,9 @@
     "'super' must be called before accessing 'this' in the constructor of a derived class.": {
         "category": "Error",
         "code": 17009
+    },
+    "Unknown typing option '{0}'.": {
+        "category": "Error",
+        "code": 17010
     }
 }

src/compiler/parser.ts

@@ -407,23 +407,6 @@ namespace ts {
         return result;
     }
 
-    /* @internal */
-    export function getScriptKindFromFileName(fileName: string): ScriptKind {
-        const ext = fileName.substr(fileName.lastIndexOf("."));
-        switch (ext.toLowerCase()) {
-            case ".js":
-                return ScriptKind.JS;
-            case ".jsx":
-                return ScriptKind.JSX;
-            case ".ts":
-                return ScriptKind.TS;
-            case ".tsx":
-                return ScriptKind.TSX;
-            default:
-                return ScriptKind.TS;
-        }
-    }
-
     // Produces a new SourceFile for the 'newText' provided. The 'textChangeRange' parameter
     // indicates what changed between the 'text' that this SourceFile has and the 'newText'.
     // The SourceFile will be created with the compiler attempting to reuse as many nodes from
@@ -551,12 +534,7 @@ namespace ts {
         let parseErrorBeforeNextFinishedNode = false;
 
         export function parseSourceFile(fileName: string, _sourceText: string, languageVersion: ScriptTarget, _syntaxCursor: IncrementalParser.SyntaxCursor, setParentNodes?: boolean, scriptKind?: ScriptKind): SourceFile {
-            // Using scriptKind as a condition handles both:
-            // - 'scriptKind' is unspecified and thus it is `undefined`
-            // - 'scriptKind' is set and it is `Unknown` (0)
-            // If the 'scriptKind' is 'undefined' or 'Unknown' then attempt
-            // to get the ScriptKind from the file name.
-            scriptKind = scriptKind ? scriptKind : getScriptKindFromFileName(fileName);
+            scriptKind = ensureScriptKind(fileName, scriptKind);
 
             initializeState(fileName, _sourceText, languageVersion, _syntaxCursor, scriptKind);
 

src/compiler/types.ts

@@ -2449,6 +2449,13 @@ namespace ts {
         [option: string]: string | number | boolean;
     }
 
+    export interface TypingOptions {
+        enableAutoDiscovery?: boolean;
+        include?: string[];
+        exclude?: string[];
+        [option: string]: any;
+    }
+
     export const enum ModuleKind {
         None = 0,
         CommonJS = 1,
@@ -2507,6 +2514,7 @@ namespace ts {
 
     export interface ParsedCommandLine {
         options: CompilerOptions;
+        typingOptions?: TypingOptions;
         fileNames: string[];
         errors: Diagnostic[];
     }

src/services/jsTyping.ts

@@ -0,0 +1,286 @@
+// Copyright (c) Microsoft. All rights reserved. Licensed under the Apache License, Version 2.0.
+// See LICENSE.txt in the project root for complete license information.
+
+/// <reference path='services.ts' />
+
+/* @internal */
+namespace ts.JsTyping {
+
+    interface TypingResolutionHost {
+        directoryExists: (path: string) => boolean;
+        fileExists: (fileName: string) => boolean;
+        readFile: (path: string, encoding?: string) => string;
+        readDirectory: (path: string, extension?: string, exclude?: string[], depth?: number) => string[];
+    };
+
+    // A map of loose file names to library names
+    // that we are confident require typings
+    let safeList: Map<string>;
+    const notFoundTypingNames: string[] = [];
+
+    function tryParseJson(jsonPath: string, host: TypingResolutionHost): any {
+        if (host.fileExists(jsonPath)) {
+            try {
+                // Strip out single-line comments
+                const contents = host.readFile(jsonPath).replace(/^\/\/(.*)$/gm, "");
+                return JSON.parse(contents);
+            }
+            catch (e) { }
+        }
+        return undefined;
+    }
+
+    function isTypingEnabled(options: TypingOptions): boolean {
+        if (options) {
+            if (options.enableAutoDiscovery ||
+                (options.include && options.include.length > 0) ||
+                (options.exclude && options.exclude.length > 0)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * @param host is the object providing I/O related operations.
+     * @param fileNames are the file names that belong to the same project.
+     * @param globalCachePath is used to get the safe list file path and as cache path if the project root path isn't specified.
+     * @param projectRootPath is the path to the project root directory. This is used for the local typings cache.
+     * @param typingOptions are used for customizing the typing inference process.
+     * @param compilerOptions are used as a source of typing inference.
+     */
+    export function discoverTypings(
+        host: TypingResolutionHost,
+        fileNames: string[],
+        globalCachePath: Path,
+        projectRootPath: Path,
+        typingOptions: TypingOptions,
+        compilerOptions: CompilerOptions)
+        : { cachedTypingPaths: string[], newTypingNames: string[], filesToWatch: string[] } {
+
+        // A typing name to typing file path mapping
+        const inferredTypings: Map<string> = {};
+
+        if (!isTypingEnabled(typingOptions)) {
+            return { cachedTypingPaths: [], newTypingNames: [], filesToWatch: [] };
+        }
+
+        const cachePath = projectRootPath ? projectRootPath : globalCachePath;
+        // Only infer typings for .js and .jsx files
+        fileNames = fileNames
+            .map(ts.normalizePath)
+            .filter(f => scriptKindIs(f, /*LanguageServiceHost*/ undefined, ScriptKind.JS, ScriptKind.JSX));
+
+        const safeListFilePath = ts.combinePaths(globalCachePath, "safeList.json");
+        if (!safeList && host.fileExists(safeListFilePath)) {
+            safeList = tryParseJson(safeListFilePath, host);
+        }
+
+        const filesToWatch: string[] = [];
+        // Directories to search for package.json, bower.json and other typing information
+        let searchDirs: string[] = [];
+        let exclude: string[] = [];
+
+        mergeTypings(typingOptions.include);
+        exclude = typingOptions.exclude ? typingOptions.exclude : [];
+
+        if (typingOptions.enableAutoDiscovery) {
+            const possibleSearchDirs = fileNames.map(ts.getDirectoryPath);
+            if (projectRootPath !== undefined) {
+                possibleSearchDirs.push(projectRootPath);
+            }
+            searchDirs = ts.deduplicate(possibleSearchDirs);
+            for (const searchDir of searchDirs) {
+                const packageJsonPath = ts.combinePaths(searchDir, "package.json");
+                getTypingNamesFromJson(packageJsonPath, filesToWatch);
+
+                const bowerJsonPath = ts.combinePaths(searchDir, "bower.json");
+                getTypingNamesFromJson(bowerJsonPath, filesToWatch);
+
+                const nodeModulesPath = ts.combinePaths(searchDir, "node_modules");
+                getTypingNamesFromNodeModuleFolder(nodeModulesPath, filesToWatch);
+            }
+
+            getTypingNamesFromSourceFileNames(fileNames);
+            getTypingNamesFromCompilerOptions(compilerOptions);
+        }
+
+        const typingsPath = ts.combinePaths(cachePath, "typings");
+        const tsdJsonPath = ts.combinePaths(cachePath, "tsd.json");
+        const tsdJsonDict = tryParseJson(tsdJsonPath, host);
+        if (tsdJsonDict) {
+            for (const notFoundTypingName of notFoundTypingNames) {
+                if (inferredTypings.hasOwnProperty(notFoundTypingName) && !inferredTypings[notFoundTypingName]) {
+                    delete inferredTypings[notFoundTypingName];
+                }
+            }
+
+            // The "installed" property in the tsd.json serves as a registry of installed typings. Each item
+            // of this object has a key of the relative file path, and a value that contains the corresponding
+            // commit hash.
+            if (hasProperty(tsdJsonDict, "installed")) {
+                for (const cachedTypingPath in tsdJsonDict.installed) {
+                    // Assuming the cachedTypingPath has the format of "[package name]/[file name]"
+                    const cachedTypingName = cachedTypingPath.substr(0, cachedTypingPath.indexOf("/"));
+                    // If the inferred[cachedTypingName] is already not null, which means we found a corresponding
+                    // d.ts file that coming with the package. That one should take higher priority.
+                    if (hasProperty(inferredTypings, cachedTypingName) && !inferredTypings[cachedTypingName]) {
+                        inferredTypings[cachedTypingName] = ts.combinePaths(typingsPath, cachedTypingPath);
+                    }
+                }
+            }
+        }
+
+        // Remove typings that the user has added to the exclude list
+        for (const excludeTypingName of exclude) {
+            delete inferredTypings[excludeTypingName];
+        }
+
+        const newTypingNames: string[] = [];
+        const cachedTypingPaths: string[] = [];
+        for (const typing in inferredTypings) {
+            if (inferredTypings[typing] !== undefined) {
+                cachedTypingPaths.push(inferredTypings[typing]);
+            }
+            else {
+                newTypingNames.push(typing);
+            }
+        }
+        return { cachedTypingPaths, newTypingNames, filesToWatch };
+
+        /**
+         * Merge a given list of typingNames to the inferredTypings map
+         */
+        function mergeTypings(typingNames: string[]) {
+            if (!typingNames) {
+                return;
+            }
+
+            for (const typing of typingNames) {
+                if (!inferredTypings.hasOwnProperty(typing)) {
+                    inferredTypings[typing] = undefined;
+                }
+            }
+        }
+
+        /**
+         * Get the typing info from common package manager json files like package.json or bower.json
+         */
+        function getTypingNamesFromJson(jsonPath: string, filesToWatch: string[]) {
+            const jsonDict = tryParseJson(jsonPath, host);
+            if (jsonDict) {
+                filesToWatch.push(jsonPath);
+                if (jsonDict.hasOwnProperty("dependencies")) {
+                    mergeTypings(Object.keys(jsonDict.dependencies));
+                }
+            }
+        }
+
+        /**
+         * Infer typing names from given file names. For example, the file name "jquery-min.2.3.4.js"
+         * should be inferred to the 'jquery' typing name; and "angular-route.1.2.3.js" should be inferred
+         * to the 'angular-route' typing name.
+         * @param fileNames are the names for source files in the project
+         */
+        function getTypingNamesFromSourceFileNames(fileNames: string[]) {
+            const jsFileNames = fileNames.filter(hasJavaScriptFileExtension);
+            const inferredTypingNames = jsFileNames.map(f => ts.removeFileExtension(ts.getBaseFileName(f.toLowerCase())));
+            const cleanedTypingNames = inferredTypingNames.map(f => f.replace(/((?:\.|-)min(?=\.|$))|((?:-|\.)\d+)/g, ""));
+            safeList === undefined ? mergeTypings(cleanedTypingNames) : mergeTypings(cleanedTypingNames.filter(f => safeList.hasOwnProperty(f)));
+
+            const jsxFileNames = fileNames.filter(f => scriptKindIs(f, /*LanguageServiceHost*/ undefined, ScriptKind.JSX));
+            if (jsxFileNames.length > 0) {
+                mergeTypings(["react"]);
+            }
+        }
+
+        /**
+         * Infer typing names from node_module folder
+         * @param nodeModulesPath is the path to the "node_modules" folder
+         */
+        function getTypingNamesFromNodeModuleFolder(nodeModulesPath: string, filesToWatch: string[]) {
+            // Todo: add support for ModuleResolutionHost too
+            if (!host.directoryExists(nodeModulesPath)) {
+                return;
+            }
+
+            const typingNames: string[] = [];
+            const packageJsonFiles =
+                host.readDirectory(nodeModulesPath, /*extension*/ undefined, /*exclude*/ undefined, /*depth*/ 2).filter(f => ts.getBaseFileName(f) === "package.json");
+            for (const packageJsonFile of packageJsonFiles) {
+                const packageJsonDict = tryParseJson(packageJsonFile, host);
+                if (!packageJsonDict) { continue; }
+
+                filesToWatch.push(packageJsonFile);
+
+                // npm 3 has the package.json contains a "_requiredBy" field
+                // we should include all the top level module names for npm 2, and only module names whose
+                // "_requiredBy" field starts with "#" or equals "/" for npm 3.
+                if (packageJsonDict._requiredBy &&
+                    packageJsonDict._requiredBy.filter((r: string) => r[0] === "#" || r === "/").length === 0) {
+                    continue;
+                }
+
+                // If the package has its own d.ts typings, those will take precedence. Otherwise the package name will be used
+                // to download d.ts files from DefinitelyTyped
+                const packageName = packageJsonDict["name"];
+                if (packageJsonDict.hasOwnProperty("typings")) {
+                    const absPath = ts.getNormalizedAbsolutePath(packageJsonDict.typings, ts.getDirectoryPath(packageJsonFile));
+                    inferredTypings[packageName] = absPath;
+                }
+                else {
+                    typingNames.push(packageName);
+                }
+            }
+            mergeTypings(typingNames);
+        }
+
+        function getTypingNamesFromCompilerOptions(options: CompilerOptions) {
+            const typingNames: string[] = [];
+            if (!options) {
+                return;
+            }
+
+            if (options.jsx === JsxEmit.React) {
+                typingNames.push("react");
+            }
+            if (options.moduleResolution === ModuleResolutionKind.NodeJs) {
+                typingNames.push("node");
+            }
+            mergeTypings(typingNames);
+        }
+    }
+
+    /**
+     * Keep a list of typings names that we know cannot be obtained at the moment (could be because
+     * of network issues or because the package doesn't hava a d.ts file in DefinitelyTyped), so
+     * that we won't try again next time within this session.
+     * @param newTypingNames The list of new typings that the host attempted to acquire
+     * @param cachePath The path to the tsd.json cache
+     * @param host The object providing I/O related operations.
+     */
+    export function updateNotFoundTypingNames(newTypingNames: string[], cachePath: string, host: TypingResolutionHost): void {
+        const tsdJsonPath = ts.combinePaths(cachePath, "tsd.json");
+        const cacheTsdJsonDict = tryParseJson(tsdJsonPath, host);
+        if (cacheTsdJsonDict) {
+            const installedTypingFiles = hasProperty(cacheTsdJsonDict, "installed")
+                ? Object.keys(cacheTsdJsonDict.installed)
+                : [];
+            const newMissingTypingNames =
+                ts.filter(newTypingNames, name => notFoundTypingNames.indexOf(name) < 0 && !isInstalled(name, installedTypingFiles));
+            for (const newMissingTypingName of newMissingTypingNames) {
+                notFoundTypingNames.push(newMissingTypingName);
+            }
+        }
+    }
+
+    function isInstalled(typing: string, installedKeys: string[]) {
+        const typingPrefix = typing + "/";
+        for (const key of installedKeys) {
+            if (key.indexOf(typingPrefix) === 0) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

src/services/services.ts

@@ -7,6 +7,7 @@
 /// <reference path='patternMatcher.ts' />
 /// <reference path='signatureHelp.ts' />
 /// <reference path='utilities.ts' />
+/// <reference path='jsTyping.ts' />
 /// <reference path='formatting\formatting.ts' />
 /// <reference path='formatting\smartIndenter.ts' />
 
@@ -1750,14 +1751,13 @@ namespace ts {
 
         private createEntry(fileName: string, path: Path) {
             let entry: HostFileInformation;
-            const scriptKind = this.host.getScriptKind ? this.host.getScriptKind(fileName) : ScriptKind.Unknown;
             const scriptSnapshot = this.host.getScriptSnapshot(fileName);
             if (scriptSnapshot) {
                 entry = {
                     hostFileName: fileName,
                     version: this.host.getScriptVersion(fileName),
                     scriptSnapshot: scriptSnapshot,
-                    scriptKind: scriptKind ? scriptKind : getScriptKindFromFileName(fileName)
+                    scriptKind: getScriptKind(fileName, this.host)
                 };
             }
 
@@ -1823,7 +1823,7 @@ namespace ts {
                 throw new Error("Could not find file: '" + fileName + "'.");
             }
 
-            const scriptKind = this.host.getScriptKind ? this.host.getScriptKind(fileName) : ScriptKind.Unknown;
+            const scriptKind = getScriptKind(fileName, this.host);
             const version = this.host.getScriptVersion(fileName);
             let sourceFile: SourceFile;
 

src/services/shims.ts

@@ -78,7 +78,7 @@ namespace ts {
          * @param exclude A JSON encoded string[] containing the paths to exclude
          *  when enumerating the directory.
          */
-        readDirectory(rootDir: string, extension: string, exclude?: string): string;
+         readDirectory(rootDir: string, extension: string, exclude?: string, depth?: number): string;
     }
 
     ///
@@ -230,6 +230,8 @@ namespace ts {
         getPreProcessedFileInfo(fileName: string, sourceText: IScriptSnapshot): string;
         getTSConfigFileInfo(fileName: string, sourceText: IScriptSnapshot): string;
         getDefaultCompilationSettings(): string;
+        resolveTypeDefinitions(fileNamesJson: string, globalCachePath: string, projectRootPath: string, typingOptionsJson: string, compilerOptionsJson: string): string;
+        updateNotFoundTypingNames(newTypingsJson: string, globalCachePath: string, projectRootPath: string): string;
     }
 
     function logInternalError(logger: Logger, err: Error) {
@@ -420,8 +422,16 @@ namespace ts {
             }
         }
 
-        public readDirectory(rootDir: string, extension: string, exclude: string[]): string[] {
-            const encoded = this.shimHost.readDirectory(rootDir, extension, JSON.stringify(exclude));
+        public readDirectory(rootDir: string, extension: string, exclude: string[], depth?: number): string[] {
+            // Wrap the API changes for 2.0 release. This try/catch
+            // should be removed once TypeScript 2.0 has shipped.
+            let encoded: string;
+            try {
+                encoded = this.shimHost.readDirectory(rootDir, extension, JSON.stringify(exclude), depth);
+            }
+            catch (e) {
+                encoded = this.shimHost.readDirectory(rootDir, extension, JSON.stringify(exclude));
+            }
             return JSON.parse(encoded);
         }
 
@@ -951,6 +961,7 @@ namespace ts {
                     if (result.error) {
                         return {
                             options: {},
+                            typingOptions: {},
                             files: [],
                             errors: [realizeDiagnostic(result.error, "\r\n")]
                         };
@@ -961,6 +972,7 @@ namespace ts {
 
                     return {
                         options: configFile.options,
+                        typingOptions: configFile.typingOptions,
                         files: configFile.fileNames,
                         errors: realizeDiagnostics(configFile.errors, "\r\n")
                     };
@@ -973,6 +985,35 @@ namespace ts {
                 () => getDefaultCompilerOptions()
             );
         }
+
+        public resolveTypeDefinitions(fileNamesJson: string, globalCachePath: string, projectRootPath: string, typingOptionsJson: string, compilerOptionsJson: string): string {
+            const getCanonicalFileName = createGetCanonicalFileName(/*useCaseSensitivefileNames:*/ false);
+            return this.forwardJSONCall("resolveTypeDefinitions()", () => {
+                const cachePath = projectRootPath ? projectRootPath : globalCachePath;
+                const typingOptions = <TypingOptions>JSON.parse(typingOptionsJson);
+                // Convert the include and exclude lists from a semi-colon delimited string to a string array
+                typingOptions.include = typingOptions.include ? typingOptions.include.toString().split(";") : [];
+                typingOptions.exclude = typingOptions.exclude ? typingOptions.exclude.toString().split(";") : [];
+
+                const compilerOptions = <CompilerOptions>JSON.parse(compilerOptionsJson);
+                const fileNames: string[] = JSON.parse(fileNamesJson);
+                return ts.JsTyping.discoverTypings(
+                    this.host,
+                    fileNames,
+                    toPath(globalCachePath, globalCachePath, getCanonicalFileName),
+                    toPath(cachePath, cachePath, getCanonicalFileName),
+                    typingOptions,
+                    compilerOptions);
+            });
+        }
+
+        public updateNotFoundTypingNames(newTypingsJson: string, globalCachePath: string, projectRootPath: string): string {
+            return this.forwardJSONCall("updateNotFoundTypingNames()", () => {
+                const newTypingNames: string[] = JSON.parse(newTypingsJson);
+                const cachePath = projectRootPath ? projectRootPath : globalCachePath;
+                ts.JsTyping.updateNotFoundTypingNames(newTypingNames, cachePath, this.host);
+            });
+        }
     }
 
     export class TypeScriptServicesFactory implements ShimFactory {

src/services/tsconfig.json

@@ -30,6 +30,7 @@
         "shims.ts",
         "signatureHelp.ts",
         "utilities.ts",
+        "jsTyping.ts",
         "formatting/formatting.ts",
         "formatting/formattingContext.ts",
         "formatting/formattingRequestKind.ts",

src/services/utilities.ts

@@ -837,4 +837,19 @@ namespace ts {
         };
         return name;
     }
+
+    export function scriptKindIs(fileName: string, host: LanguageServiceHost, ...scriptKinds: ScriptKind[]): boolean {
+        const scriptKind = getScriptKind(fileName, host);
+        return forEach(scriptKinds, k => k === scriptKind);
+    }
+
+    export function getScriptKind(fileName: string, host?: LanguageServiceHost): ScriptKind {
+        // First check to see if the script kind can be determined from the file name
+        var scriptKind = getScriptKindFromFileName(fileName);
+        if (scriptKind === ScriptKind.Unknown && host && host.getScriptKind) {
+            // Next check to see if the host can resolve the script kind
+            scriptKind = host.getScriptKind(fileName);
+        }
+        return ensureScriptKind(fileName, scriptKind);
+    }
 }