Add syntactic classification for doc comments.

src/compiler/parser.ts

@@ -363,9 +363,9 @@ module ts {
         }
     }
 
-    export function createSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, setParentNodes = false): SourceFile {
+    export function createSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, setParentNodes = false, includeDocComments = false): SourceFile {
         let start = new Date().getTime();
-        let result = Parser.parseSourceFile(fileName, sourceText, languageVersion, /*syntaxCursor*/ undefined, setParentNodes);
+        let result = Parser.parseSourceFile(fileName, sourceText, languageVersion, /*syntaxCursor*/ undefined, setParentNodes, includeDocComments);
 
         parseTime += new Date().getTime() - start;
         return result;
@@ -493,10 +493,10 @@ module ts {
         // attached to the EOF token.
         let parseErrorBeforeNextFinishedNode: boolean = false;
 
-        export function parseSourceFile(fileName: string, _sourceText: string, languageVersion: ScriptTarget, _syntaxCursor: IncrementalParser.SyntaxCursor, setParentNodes?: boolean): SourceFile {
+        export function parseSourceFile(fileName: string, _sourceText: string, languageVersion: ScriptTarget, _syntaxCursor: IncrementalParser.SyntaxCursor, setParentNodes?: boolean, includeDocComments?: boolean): SourceFile {
             initializeState(fileName, _sourceText, languageVersion, _syntaxCursor);
 
-            let result = parseSourceFileWorker(fileName, languageVersion, setParentNodes);
+            let result = parseSourceFileWorker(fileName, languageVersion, setParentNodes, includeDocComments);
 
             clearState();
 
@@ -535,7 +535,7 @@ module ts {
             sourceText = undefined;
         }
 
-        function parseSourceFileWorker(fileName: string, languageVersion: ScriptTarget, setParentNodes: boolean): SourceFile {
+        function parseSourceFileWorker(fileName: string, languageVersion: ScriptTarget, setParentNodes: boolean, includeDocComments: boolean): SourceFile {
             sourceFile = createSourceFile(fileName, languageVersion);
 
             // Prime the scanner.
@@ -560,7 +560,7 @@ module ts {
             // If this is a javascript file, proactively see if we can get JSDoc comments for 
             // relevant nodes in the file.  We'll use these to provide typing informaion if they're
             // available.
-            if (isJavaScript(fileName)) {
+            if (includeDocComments || isJavaScript(fileName)) {
                 addJSDocComments();
             }
 

src/harness/harness.ts

@@ -167,7 +167,7 @@ module Utils {
                     continue;
                 }
                 var child = (<any>node)[childName];
-                if (isNodeOrArray(child)) {
+                if (isNodeOrArray(child) && childName !== "jsDocComment") {
                     assert.isFalse(childNodesAndArrays.indexOf(child) < 0,
                         "Missing child when forEach'ing over node: " + (<any>ts).SyntaxKind[node.kind] + "-" + childName);
                 }

src/services/services.ts

@@ -1510,6 +1510,7 @@ module ts {
         public static typeParameterName = "type parameter name";
         public static typeAliasName = "type alias name";
         public static parameterName = "parameter name";
+        public static docCommentTagName = "doc comment tag name";
     }
 
     export const enum ClassificationType {
@@ -1529,7 +1530,8 @@ module ts {
         moduleName = 14,
         typeParameterName = 15,
         typeAliasName = 16,
-        parameterName = 17
+        parameterName = 17,
+        docCommentTagName = 18,
     }
 
     /// Language Service
@@ -1813,7 +1815,8 @@ module ts {
     }
 
     export function createLanguageServiceSourceFile(fileName: string, scriptSnapshot: IScriptSnapshot, scriptTarget: ScriptTarget, version: string, setNodeParents: boolean): SourceFile {
-        let sourceFile = createSourceFile(fileName, scriptSnapshot.getText(0, scriptSnapshot.getLength()), scriptTarget, setNodeParents);
+        let text = scriptSnapshot.getText(0, scriptSnapshot.getLength());
+        let sourceFile = createSourceFile(fileName, text, scriptTarget, setNodeParents, /*includeDocComments:*/ true);
         setSourceFileFields(sourceFile, scriptSnapshot, version);
         // after full parsing we can use table with interned strings as name table
         sourceFile.nameTable = sourceFile.identifiers;
@@ -2832,6 +2835,7 @@ module ts {
             let typeChecker = program.getTypeChecker();
             let syntacticStart = new Date().getTime();
             let sourceFile = getValidSourceFile(fileName);
+            let isJavaScriptFile = isJavaScript(fileName);
 
             let start = new Date().getTime();
             let currentToken = getTokenAtPosition(sourceFile, position);
@@ -2932,13 +2936,29 @@ module ts {
                 }
 
                 let type = typeChecker.getTypeAtLocation(node);
+                addTypeProperties(type);
+            }
+
+            function addTypeProperties(type: Type) {
                 if (type) {
                     // Filter private properties
-                    forEach(type.getApparentProperties(), symbol => {
+                    for (let symbol of type.getApparentProperties()) {
                         if (typeChecker.isValidPropertyAccess(<PropertyAccessExpression>(node.parent), symbol.name)) {
                             symbols.push(symbol);
                         }
-                    });
+                    }
+
+                    if (isJavaScriptFile && type.flags & TypeFlags.Union) {
+                        // In javascript files, for union types, we don't just get the members that 
+                        // the individual types have in common, we also include all the members that
+                        // each individual type has.  This is because we're going to add all identifiers
+                        // anyways.  So we might as well elevate the members that were at least part
+                        // of the individual types to a higher status than since we know what they are.
+                        let unionType = <UnionType>type;
+                        for (let elementType of unionType.types) {
+                            addTypeProperties(elementType);
+                        }
+                    }
                 }
             }
 
@@ -6031,6 +6051,7 @@ module ts {
                 case ClassificationType.typeParameterName: return ClassificationTypeNames.typeParameterName;
                 case ClassificationType.typeAliasName: return ClassificationTypeNames.typeAliasName;
                 case ClassificationType.parameterName: return ClassificationTypeNames.parameterName;
+                case ClassificationType.docCommentTagName: return ClassificationTypeNames.docCommentTagName;
             }
         }
 
@@ -6093,8 +6114,7 @@ module ts {
                     // Only bother with the trivia if it at least intersects the span of interest.
                     if (textSpanIntersectsWith(span, start, width)) {
                         if (isComment(kind)) {
-                            // Simple comment.  Just add as is.
-                            pushClassification(start, width, ClassificationType.comment);
+                            classifyComment(token, kind, start, width);
                             continue;
                         }
 
@@ -6118,6 +6138,90 @@ module ts {
                 }
             }
 
+            function classifyComment(token: Node, kind: SyntaxKind, start: number, width: number) {
+                if (kind === SyntaxKind.MultiLineCommentTrivia) {
+                    // See if this is a doc comment.  If so, we'll classify certain portions of it
+                    // specially.
+                    let jsDocComment = parseIsolatedJSDocComment(sourceFile.text, start, width);
+                    if (jsDocComment && jsDocComment.jsDocComment) {
+                        jsDocComment.jsDocComment.parent = token;
+                        classifyJSDocComment(jsDocComment.jsDocComment);
+                        return;
+                    }
+                }
+
+                // Simple comment.  Just add as is.
+                pushCommentRange(start, width);
+            }
+
+            function pushCommentRange(start: number, width: number) {
+                pushClassification(start, width, ClassificationType.comment);
+            }
+
+            function classifyJSDocComment(docComment: JSDocComment) {
+                let pos = docComment.pos;
+
+                for (let tag of docComment.tags) {
+                    if (tag.pos !== pos) {
+                        pushCommentRange(pos, tag.pos - pos);
+                    }
+
+                    pushClassification(tag.atToken.pos, tag.atToken.end - tag.atToken.pos, ClassificationType.punctuation);
+                    pushClassification(tag.tagName.pos, tag.tagName.end - tag.tagName.pos, ClassificationType.docCommentTagName);
+
+                    pos = tag.tagName.end;
+
+                    switch (tag.kind) {
+                        case SyntaxKind.JSDocParameterTag:
+                            processJSDocParameterTag(<JSDocParameterTag>tag);
+                            break;
+                        case SyntaxKind.JSDocTemplateTag:
+                            processJSDocTemplateTag(<JSDocTemplateTag>tag);
+                            break;
+                        case SyntaxKind.JSDocTypeTag:
+                            processElement((<JSDocTypeTag>tag).typeExpression);
+                            break;
+                        case SyntaxKind.JSDocReturnTag:
+                            processElement((<JSDocReturnTag>tag).typeExpression);
+                            break;
+                    }
+
+                    pos = tag.end;
+                }
+
+                if (pos !== docComment.end) {
+                    pushCommentRange(pos, docComment.end - pos);
+                }
+
+                return;
+
+                function processJSDocParameterTag(tag: JSDocParameterTag) {
+                    if (tag.preParameterName) {
+                        pushCommentRange(pos, tag.preParameterName.pos - pos);
+                        pushClassification(tag.preParameterName.pos, tag.preParameterName.end - tag.preParameterName.pos, ClassificationType.parameterName);
+                        pos = tag.preParameterName.end;
+                    }
+
+                    if (tag.typeExpression) {
+                        pushCommentRange(pos, tag.typeExpression.pos - pos);
+                        processElement(tag.typeExpression);
+                        pos = tag.typeExpression.end;
+                    }
+
+                    if (tag.postParameterName) {
+                        pushCommentRange(pos, tag.postParameterName.pos - pos);
+                        pushClassification(tag.postParameterName.pos, tag.postParameterName.end - tag.postParameterName.pos, ClassificationType.parameterName);
+                        pos = tag.postParameterName.end;
+                    }
+                }
+            }
+
+            function processJSDocTemplateTag(tag: JSDocTemplateTag) {
+                for (let child of tag.getChildren()) {
+                    processElement(child);
+                }
+            }
+
             function classifyDisabledMergeCode(text: string, start: number, end: number) {
                 // Classify the line that the ======= marker is on as a comment.  Then just lex 
                 // all further tokens and add them to the result.
@@ -6251,9 +6355,13 @@ module ts {
             }
 
             function processElement(element: Node) {
+                if (!element) {
+                    return;
+                }
+
                 // Ignore nodes that don't intersect the original span to classify.
                 if (textSpanIntersectsWith(span, element.getFullStart(), element.getFullWidth())) {
-                    let children = element.getChildren();
+                    let children = element.getChildren(sourceFile);
                     for (let child of children) {
                         if (isToken(child)) {
                             classifyToken(child);

tests/cases/fourslash/fourslash.ts

@@ -639,6 +639,10 @@ module FourSlashInterface {
             return getClassification("punctuation", text, position);
         }
 
+        export function docCommentTagName(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
+            return getClassification("docCommentTagName", text, position);
+        }
+
         export function className(text: string, position?: number): { classificationType: string; text: string; textSpan?: TextSpan } {
             return getClassification("className", text, position);
         }

tests/cases/fourslash/syntacticClassificationsDocComment1.ts

@@ -0,0 +1,17 @@
+/// <reference path="fourslash.ts"/>
+
+//// /** @type {number} */
+//// var v;
+
+var c = classification;
+verify.syntacticClassificationsAre(
+    c.comment("/** "),
+    c.punctuation("@"),
+    c.docCommentTagName("type"),
+    c.punctuation("{"),
+    c.keyword("number"),
+    c.punctuation("}"),
+    c.comment(" */"),
+    c.keyword("var"),
+    c.text("v"),
+    c.punctuation(";"));