Merge pull request #7179 from Microsoft/jsTypingForAcquireDts

Salsa: JS support for discovering and acquiring d.ts files

Jakefile.js

@@ -927,6 +927,7 @@ var servicesLintTargets = [
     "patternMatcher.ts",
     "services.ts",
     "shims.ts",
+    "jsTyping.ts"
 ].map(function (s) {
     return path.join(servicesDirectory, s);
 });

src/compiler/commandLineParser.ts

@@ -543,6 +543,7 @@ namespace ts {
         return {
             options,
             fileNames: getFileNames(),
+            typingOptions: getTypingOptions(),
             errors
         };
 
@@ -607,6 +608,35 @@ namespace ts {
             }
             return fileNames;
         }
+
+        function getTypingOptions(): TypingOptions {
+            const options: TypingOptions = getBaseFileName(configFileName) === "jsconfig.json"
+                ? { enableAutoDiscovery: true, include: [], exclude: [] }
+                : { enableAutoDiscovery: false, include: [], exclude: [] };
+            const jsonTypingOptions = json["typingOptions"];
+            if (jsonTypingOptions) {
+                for (const id in jsonTypingOptions) {
+                    if (id === "enableAutoDiscovery") {
+                        if (typeof jsonTypingOptions[id] === "boolean") {
+                            options.enableAutoDiscovery = jsonTypingOptions[id];
+                        }
+                        else {
+                            errors.push(createCompilerDiagnostic(Diagnostics.Unknown_typing_option_0, id));
+                        }
+                    }
+                    else if (id === "include") {
+                        options.include = convertJsonOptionToStringArray(id, jsonTypingOptions[id], errors);
+                    }
+                    else if (id === "exclude") {
+                        options.exclude = convertJsonOptionToStringArray(id, jsonTypingOptions[id], errors);
+                    }
+                    else {
+                        errors.push(createCompilerDiagnostic(Diagnostics.Unknown_typing_option_0, id));
+                    }
+                }
+            }
+            return options;
+        }
     }
 
     export function convertCompilerOptionsFromJson(jsonOptions: any, basePath: string, configFileName?: string): { options: CompilerOptions, errors: Diagnostic[] } {
@@ -647,28 +677,7 @@ namespace ts {
                                 break;
                             case "object":
                                 // "object" options with 'isFilePath' = true expected to be string arrays
-                                let paths: string[] = [];
-                                let invalidOptionType = false;
-                                if (!isArray(value)) {
-                                    invalidOptionType = true;
-                                }
-                                else {
-                                    for (const element of <any[]>value) {
-                                        if (typeof element === "string") {
-                                            paths.push(normalizePath(combinePaths(basePath, element)));
-                                        }
-                                        else {
-                                            invalidOptionType = true;
-                                            break;
-                                        }
-                                    }
-                                }
-                                if (invalidOptionType) {
-                                    errors.push(createCompilerDiagnostic(Diagnostics.Option_0_should_have_array_of_strings_as_a_value, opt.name));
-                                }
-                                else {
-                                    value = paths;
-                                }
+                                value = convertJsonOptionToStringArray(opt.name, value, errors, (element) => normalizePath(combinePaths(basePath, element)));
                                 break;
                         }
                         if (value === "") {
@@ -688,4 +697,28 @@ namespace ts {
 
         return { options, errors };
     }
+
+    function convertJsonOptionToStringArray(optionName: string, optionJson: any, errors: Diagnostic[], func?: (element: string) => string): string[] {
+        const items: string[] = [];
+        let invalidOptionType = false;
+        if (!isArray(optionJson)) {
+            invalidOptionType = true;
+        }
+        else {
+            for (const element of <any[]>optionJson) {
+                if (typeof element === "string") {
+                    const item = func ? func(element) : element;
+                    items.push(item);
+                }
+                else {
+                    invalidOptionType = true;
+                    break;
+                }
+            }
+        }
+        if (invalidOptionType) {
+            errors.push(createCompilerDiagnostic(Diagnostics.Option_0_should_have_array_of_strings_as_a_value, optionName));
+        }
+        return items;
+    }
 }

src/compiler/core.ts

@@ -278,6 +278,14 @@ namespace ts {
         return hasOwnProperty.call(map, key);
     }
 
+    export function getKeys<T>(map: Map<T>): string[] {
+        const keys: string[] = [];
+        for (const key in map) {
+            keys.push(key);
+        }
+        return keys;
+    }
+
     export function getProperty<T>(map: Map<T>, key: string): T {
         return hasOwnProperty.call(map, key) ? map[key] : undefined;
     }
@@ -778,6 +786,32 @@ namespace ts {
         return pathLen > extLen && path.substr(pathLen - extLen, extLen) === extension;
     }
 
+    export function ensureScriptKind(fileName: string, scriptKind?: ScriptKind): ScriptKind {
+        // Using scriptKind as a condition handles both:
+        // - 'scriptKind' is unspecified and thus it is `undefined`
+        // - 'scriptKind' is set and it is `Unknown` (0)
+        // If the 'scriptKind' is 'undefined' or 'Unknown' then we attempt
+        // to get the ScriptKind from the file name. If it cannot be resolved
+        // from the file name then the default 'TS' script kind is returned.
+        return (scriptKind || getScriptKindFromFileName(fileName)) || ScriptKind.TS;
+    }
+
+    export function getScriptKindFromFileName(fileName: string): ScriptKind {
+        const ext = fileName.substr(fileName.lastIndexOf("."));
+        switch (ext.toLowerCase()) {
+            case ".js":
+                return ScriptKind.JS;
+            case ".jsx":
+                return ScriptKind.JSX;
+            case ".ts":
+                return ScriptKind.TS;
+            case ".tsx":
+                return ScriptKind.TSX;
+            default:
+                return ScriptKind.Unknown;
+        }
+    }
+
     /**
      *  List of supported extensions in order of file resolution precedence.
      */

src/compiler/diagnosticMessages.json

@@ -2804,5 +2804,9 @@
     "'super' must be called before accessing 'this' in the constructor of a derived class.": {
         "category": "Error",
         "code": 17009
+    },
+    "Unknown typing option '{0}'.": {
+        "category": "Error",
+        "code": 17010
     }
 }

src/compiler/parser.ts

@@ -407,23 +407,6 @@ namespace ts {
         return result;
     }
 
-    /* @internal */
-    export function getScriptKindFromFileName(fileName: string): ScriptKind {
-        const ext = fileName.substr(fileName.lastIndexOf("."));
-        switch (ext.toLowerCase()) {
-            case ".js":
-                return ScriptKind.JS;
-            case ".jsx":
-                return ScriptKind.JSX;
-            case ".ts":
-                return ScriptKind.TS;
-            case ".tsx":
-                return ScriptKind.TSX;
-            default:
-                return ScriptKind.TS;
-        }
-    }
-
     // Produces a new SourceFile for the 'newText' provided. The 'textChangeRange' parameter
     // indicates what changed between the 'text' that this SourceFile has and the 'newText'.
     // The SourceFile will be created with the compiler attempting to reuse as many nodes from
@@ -551,12 +534,7 @@ namespace ts {
         let parseErrorBeforeNextFinishedNode = false;
 
         export function parseSourceFile(fileName: string, _sourceText: string, languageVersion: ScriptTarget, _syntaxCursor: IncrementalParser.SyntaxCursor, setParentNodes?: boolean, scriptKind?: ScriptKind): SourceFile {
-            // Using scriptKind as a condition handles both:
-            // - 'scriptKind' is unspecified and thus it is `undefined`
-            // - 'scriptKind' is set and it is `Unknown` (0)
-            // If the 'scriptKind' is 'undefined' or 'Unknown' then attempt
-            // to get the ScriptKind from the file name.
-            scriptKind = scriptKind ? scriptKind : getScriptKindFromFileName(fileName);
+            scriptKind = ensureScriptKind(fileName, scriptKind);
 
             initializeState(fileName, _sourceText, languageVersion, _syntaxCursor, scriptKind);
 

src/compiler/types.ts

@@ -2433,6 +2433,23 @@ namespace ts {
         [option: string]: string | number | boolean | TsConfigOnlyOptions;
     }
 
+    export interface TypingOptions {
+        enableAutoDiscovery?: boolean;
+        include?: string[];
+        exclude?: string[];
+        [option: string]: string[] | boolean;
+    }
+
+    export interface DiscoverTypingsInfo {
+        fileNames: string[];                            // The file names that belong to the same project.
+        cachePath: string;                              // The path to the typings cache
+        projectRootPath: string;                        // The path to the project root directory
+        safeListPath: string;                           // The path used to retrieve the safe list
+        packageNameToTypingLocation: Map<string>;       // The map of package names to their cached typing locations
+        typingOptions: TypingOptions;                   // Used to customize the typing inference process
+        compilerOptions: CompilerOptions;               // Used as a source for typing inference
+    }
+
     export enum ModuleKind {
         None = 0,
         CommonJS = 1,
@@ -2491,6 +2508,7 @@ namespace ts {
 
     export interface ParsedCommandLine {
         options: CompilerOptions;
+        typingOptions?: TypingOptions;
         fileNames: string[];
         errors: Diagnostic[];
     }

src/services/jsTyping.ts

@@ -0,0 +1,229 @@
+// Copyright (c) Microsoft. All rights reserved. Licensed under the Apache License, Version 2.0.
+// See LICENSE.txt in the project root for complete license information.
+
+/// <reference path='services.ts' />
+
+/* @internal */
+namespace ts.JsTyping {
+
+    interface TypingResolutionHost {
+        directoryExists: (path: string) => boolean;
+        fileExists: (fileName: string) => boolean;
+        readFile: (path: string, encoding?: string) => string;
+        readDirectory: (path: string, extension?: string, exclude?: string[], depth?: number) => string[];
+    };
+
+    interface PackageJson {
+        _requiredBy?: string[];
+        dependencies?: Map<string>;
+        devDependencies?: Map<string>;
+        name?: string;
+        optionalDependencies?: Map<string>;
+        peerDependencies?: Map<string>;
+        typings?: string;
+    };
+
+    // A map of loose file names to library names
+    // that we are confident require typings
+    let safeList: Map<string>;
+
+    /**
+     * @param host is the object providing I/O related operations.
+     * @param fileNames are the file names that belong to the same project
+     * @param cachePath is the path to the typings cache
+     * @param projectRootPath is the path to the project root directory
+     * @param safeListPath is the path used to retrieve the safe list
+     * @param packageNameToTypingLocation is the map of package names to their cached typing locations
+     * @param typingOptions are used to customize the typing inference process
+     * @param compilerOptions are used as a source for typing inference
+     */
+    export function discoverTypings(
+        host: TypingResolutionHost,
+        fileNames: string[],
+        cachePath: Path,
+        projectRootPath: Path,
+        safeListPath: Path,
+        packageNameToTypingLocation: Map<string>,
+        typingOptions: TypingOptions,
+        compilerOptions: CompilerOptions):
+        { cachedTypingPaths: string[], newTypingNames: string[], filesToWatch: string[] } {
+
+        // A typing name to typing file path mapping
+        const inferredTypings: Map<string> = {};
+
+        if (!typingOptions || !typingOptions.enableAutoDiscovery) {
+            return { cachedTypingPaths: [], newTypingNames: [], filesToWatch: [] };
+        }
+
+        // Only infer typings for .js and .jsx files
+        fileNames = filter(map(fileNames, normalizePath), f => scriptKindIs(f, /*LanguageServiceHost*/ undefined, ScriptKind.JS, ScriptKind.JSX));
+
+        if (!safeList) {
+            const result = readConfigFile(safeListPath, (path: string) => host.readFile(path));
+            if (result.config) {
+                safeList = result.config;
+            }
+            else {
+                safeList = {};
+            };
+        }
+
+        const filesToWatch: string[] = [];
+        // Directories to search for package.json, bower.json and other typing information
+        let searchDirs: string[] = [];
+        let exclude: string[] = [];
+
+        mergeTypings(typingOptions.include);
+        exclude = typingOptions.exclude || [];
+
+        const possibleSearchDirs = map(fileNames, getDirectoryPath);
+        if (projectRootPath !== undefined) {
+            possibleSearchDirs.push(projectRootPath);
+        }
+        searchDirs = deduplicate(possibleSearchDirs);
+        for (const searchDir of searchDirs) {
+            const packageJsonPath = combinePaths(searchDir, "package.json");
+            getTypingNamesFromJson(packageJsonPath, filesToWatch);
+
+            const bowerJsonPath = combinePaths(searchDir, "bower.json");
+            getTypingNamesFromJson(bowerJsonPath, filesToWatch);
+
+            const nodeModulesPath = combinePaths(searchDir, "node_modules");
+            getTypingNamesFromNodeModuleFolder(nodeModulesPath);
+        }
+        getTypingNamesFromSourceFileNames(fileNames);
+
+        // Add the cached typing locations for inferred typings that are already installed
+        for (const name in packageNameToTypingLocation) {
+            if (hasProperty(inferredTypings, name) && !inferredTypings[name]) {
+                inferredTypings[name] = packageNameToTypingLocation[name];
+            }
+        }
+
+        // Remove typings that the user has added to the exclude list
+        for (const excludeTypingName of exclude) {
+            delete inferredTypings[excludeTypingName];
+        }
+
+        const newTypingNames: string[] = [];
+        const cachedTypingPaths: string[] = [];
+        for (const typing in inferredTypings) {
+            if (inferredTypings[typing] !== undefined) {
+                cachedTypingPaths.push(inferredTypings[typing]);
+            }
+            else {
+                newTypingNames.push(typing);
+            }
+        }
+        return { cachedTypingPaths, newTypingNames, filesToWatch };
+
+        /**
+         * Merge a given list of typingNames to the inferredTypings map
+         */
+        function mergeTypings(typingNames: string[]) {
+            if (!typingNames) {
+                return;
+            }
+
+            for (const typing of typingNames) {
+                if (!hasProperty(inferredTypings, typing)) {
+                    inferredTypings[typing] = undefined;
+                }
+            }
+        }
+
+        /**
+         * Get the typing info from common package manager json files like package.json or bower.json
+         */
+        function getTypingNamesFromJson(jsonPath: string, filesToWatch: string[]) {
+            const result = readConfigFile(jsonPath, (path: string) => host.readFile(path));
+            if (result.config) {
+                const jsonConfig: PackageJson = result.config;
+                filesToWatch.push(jsonPath);
+                if (jsonConfig.dependencies) {
+                    mergeTypings(getKeys(jsonConfig.dependencies));
+                }
+                if (jsonConfig.devDependencies) {
+                    mergeTypings(getKeys(jsonConfig.devDependencies));
+                }
+                if (jsonConfig.optionalDependencies) {
+                    mergeTypings(getKeys(jsonConfig.optionalDependencies));
+                }
+                if (jsonConfig.peerDependencies) {
+                    mergeTypings(getKeys(jsonConfig.peerDependencies));
+                }
+            }
+        }
+
+        /**
+         * Infer typing names from given file names. For example, the file name "jquery-min.2.3.4.js"
+         * should be inferred to the 'jquery' typing name; and "angular-route.1.2.3.js" should be inferred
+         * to the 'angular-route' typing name.
+         * @param fileNames are the names for source files in the project
+         */
+        function getTypingNamesFromSourceFileNames(fileNames: string[]) {
+            const jsFileNames = filter(fileNames, hasJavaScriptFileExtension);
+            const inferredTypingNames = map(jsFileNames, f => removeFileExtension(getBaseFileName(f.toLowerCase())));
+            const cleanedTypingNames = map(inferredTypingNames, f => f.replace(/((?:\.|-)min(?=\.|$))|((?:-|\.)\d+)/g, ""));
+            if (safeList === undefined) {
+                mergeTypings(cleanedTypingNames);
+            }
+            else {
+                mergeTypings(filter(cleanedTypingNames, f => hasProperty(safeList, f)));
+            }
+
+            const hasJsxFile = forEach(fileNames, f => scriptKindIs(f, /*LanguageServiceHost*/ undefined, ScriptKind.JSX));
+            if (hasJsxFile) {
+                mergeTypings(["react"]);
+            }
+        }
+
+        /**
+         * Infer typing names from node_module folder
+         * @param nodeModulesPath is the path to the "node_modules" folder
+         */
+        function getTypingNamesFromNodeModuleFolder(nodeModulesPath: string) {
+            // Todo: add support for ModuleResolutionHost too
+            if (!host.directoryExists(nodeModulesPath)) {
+                return;
+            }
+
+            const typingNames: string[] = [];
+            const fileNames = host.readDirectory(nodeModulesPath, "*.json", /*exclude*/ undefined, /*depth*/ 2);
+            for (const fileName of fileNames) {
+                const normalizedFileName = normalizePath(fileName);
+                if (getBaseFileName(normalizedFileName) !== "package.json") {
+                    continue;
+                }
+                const result = readConfigFile(normalizedFileName, (path: string) => host.readFile(path));
+                if (!result.config) {
+                    continue;
+                }
+                const packageJson: PackageJson = result.config;
+
+                // npm 3's package.json contains a "_requiredBy" field
+                // we should include all the top level module names for npm 2, and only module names whose
+                // "_requiredBy" field starts with "#" or equals "/" for npm 3.
+                if (packageJson._requiredBy &&
+                    filter(packageJson._requiredBy, (r: string) => r[0] === "#" || r === "/").length === 0) {
+                    continue;
+                }
+
+                // If the package has its own d.ts typings, those will take precedence. Otherwise the package name will be used
+                // to download d.ts files from DefinitelyTyped
+                if (!packageJson.name) {
+                    continue;
+                }
+                if (packageJson.typings) {
+                    const absolutePath = getNormalizedAbsolutePath(packageJson.typings, getDirectoryPath(normalizedFileName));
+                    inferredTypings[packageJson.name] = absolutePath;
+                }
+                else {
+                    typingNames.push(packageJson.name);
+                }
+            }
+            mergeTypings(typingNames);
+        }
+
+    }
+}

src/services/services.ts

@@ -7,6 +7,7 @@
 /// <reference path='patternMatcher.ts' />
 /// <reference path='signatureHelp.ts' />
 /// <reference path='utilities.ts' />
+/// <reference path='jsTyping.ts' />
 /// <reference path='formatting\formatting.ts' />
 /// <reference path='formatting\smartIndenter.ts' />
 
@@ -1751,14 +1752,13 @@ namespace ts {
 
         private createEntry(fileName: string, path: Path) {
             let entry: HostFileInformation;
-            const scriptKind = this.host.getScriptKind ? this.host.getScriptKind(fileName) : ScriptKind.Unknown;
             const scriptSnapshot = this.host.getScriptSnapshot(fileName);
             if (scriptSnapshot) {
                 entry = {
                     hostFileName: fileName,
                     version: this.host.getScriptVersion(fileName),
                     scriptSnapshot: scriptSnapshot,
-                    scriptKind: scriptKind ? scriptKind : getScriptKindFromFileName(fileName)
+                    scriptKind: getScriptKind(fileName, this.host)
                 };
             }
 
@@ -1824,7 +1824,7 @@ namespace ts {
                 throw new Error("Could not find file: '" + fileName + "'.");
             }
 
-            const scriptKind = this.host.getScriptKind ? this.host.getScriptKind(fileName) : ScriptKind.Unknown;
+            const scriptKind = getScriptKind(fileName, this.host);
             const version = this.host.getScriptVersion(fileName);
             let sourceFile: SourceFile;
 

src/services/shims.ts

@@ -78,7 +78,7 @@ namespace ts {
          * @param exclude A JSON encoded string[] containing the paths to exclude
          *  when enumerating the directory.
          */
-        readDirectory(rootDir: string, extension: string, exclude?: string): string;
+        readDirectory(rootDir: string, extension: string, exclude?: string, depth?: number): string;
 
         trace(s: string): void;
     }
@@ -232,6 +232,7 @@ namespace ts {
         getPreProcessedFileInfo(fileName: string, sourceText: IScriptSnapshot): string;
         getTSConfigFileInfo(fileName: string, sourceText: IScriptSnapshot): string;
         getDefaultCompilationSettings(): string;
+        discoverTypings(discoverTypingsJson: string): string;
     }
 
     function logInternalError(logger: Logger, err: Error) {
@@ -422,8 +423,16 @@ namespace ts {
             }
         }
 
-        public readDirectory(rootDir: string, extension: string, exclude: string[]): string[] {
-            const encoded = this.shimHost.readDirectory(rootDir, extension, JSON.stringify(exclude));
+        public readDirectory(rootDir: string, extension: string, exclude: string[], depth?: number): string[] {
+            // Wrap the API changes for 2.0 release. This try/catch
+            // should be removed once TypeScript 2.0 has shipped.
+            let encoded: string;
+            try {
+                encoded = this.shimHost.readDirectory(rootDir, extension, JSON.stringify(exclude), depth);
+            }
+            catch (e) {
+                encoded = this.shimHost.readDirectory(rootDir, extension, JSON.stringify(exclude));
+            }
             return JSON.parse(encoded);
         }
 
@@ -953,6 +962,7 @@ namespace ts {
                     if (result.error) {
                         return {
                             options: {},
+                            typingOptions: {},
                             files: [],
                             errors: [realizeDiagnostic(result.error, "\r\n")]
                         };
@@ -963,6 +973,7 @@ namespace ts {
 
                     return {
                         options: configFile.options,
+                        typingOptions: configFile.typingOptions,
                         files: configFile.fileNames,
                         errors: realizeDiagnostics(configFile.errors, "\r\n")
                     };
@@ -975,6 +986,22 @@ namespace ts {
                 () => getDefaultCompilerOptions()
             );
         }
+
+        public discoverTypings(discoverTypingsJson: string): string {
+            const getCanonicalFileName = createGetCanonicalFileName(/*useCaseSensitivefileNames:*/ false);
+            return this.forwardJSONCall("discoverTypings()", () => {
+                const info = <DiscoverTypingsInfo>JSON.parse(discoverTypingsJson);
+                return ts.JsTyping.discoverTypings(
+                    this.host,
+                    info.fileNames,
+                    toPath(info.cachePath, info.cachePath, getCanonicalFileName),
+                    toPath(info.projectRootPath, info.projectRootPath, getCanonicalFileName),
+                    toPath(info.safeListPath, info.safeListPath, getCanonicalFileName),
+                    info.packageNameToTypingLocation,
+                    info.typingOptions,
+                    info.compilerOptions);
+            });
+        }
     }
 
     export class TypeScriptServicesFactory implements ShimFactory {

src/services/tsconfig.json

@@ -29,6 +29,7 @@
         "shims.ts",
         "signatureHelp.ts",
         "utilities.ts",
+        "jsTyping.ts",
         "formatting/formatting.ts",
         "formatting/formattingContext.ts",
         "formatting/formattingRequestKind.ts",

src/services/utilities.ts

@@ -837,4 +837,19 @@ namespace ts {
         };
         return name;
     }
+
+    export function scriptKindIs(fileName: string, host: LanguageServiceHost, ...scriptKinds: ScriptKind[]): boolean {
+        const scriptKind = getScriptKind(fileName, host);
+        return forEach(scriptKinds, k => k === scriptKind);
+    }
+
+    export function getScriptKind(fileName: string, host?: LanguageServiceHost): ScriptKind {
+        // First check to see if the script kind can be determined from the file name
+        var scriptKind = getScriptKindFromFileName(fileName);
+        if (scriptKind === ScriptKind.Unknown && host && host.getScriptKind) {
+            // Next check to see if the host can resolve the script kind
+            scriptKind = host.getScriptKind(fileName);
+        }
+        return ensureScriptKind(fileName, scriptKind);
+    }
 }