Add a common, dense, format for classification operations to lower cost of processing on the host side.

We now just return an array of triples to represent classified results.  The triple contains:
1) the start of the classification.
2) the length of the classification.
3) the type of the clasification.

We also encode this into a comma separated string when passing over to the managed side
(as opposed to an JSON array).  That way we don't pay such a high JSON parsing cost.
Instead, we can just do a string.split(",") on the encoded triples and process each
element ourselves.

src/harness/harnessLanguageService.ts

@@ -235,6 +235,9 @@ module Harness.LanguageService {
     class ClassifierShimProxy implements ts.Classifier { 
         constructor(private shim: ts.ClassifierShim) {
         }
+        getLexicalClassifications2(text: string, lexState: ts.EndOfLineState, classifyKeywordsInGenerics?: boolean): ts.Classifications {
+            throw new Error("NYI");
+        }
         getClassificationsForLine(text: string, lexState: ts.EndOfLineState, classifyKeywordsInGenerics?: boolean): ts.ClassificationResult {
             var result = this.shim.getClassificationsForLine(text, lexState, classifyKeywordsInGenerics).split('\n');
             var entries: ts.ClassificationInfo[] = [];
@@ -300,10 +303,10 @@ module Harness.LanguageService {
         getSemanticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] {
             return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length));
         }
-        getSyntacticClassifications2(fileName: string, span: ts.TextSpan): number[] {
+        getSyntacticClassifications2(fileName: string, span: ts.TextSpan): ts.Classifications {
             return unwrapJSONCallResult(this.shim.getSyntacticClassifications2(fileName, span.start, span.length));
         }
-        getSemanticClassifications2(fileName: string, span: ts.TextSpan): number[] {
+        getSemanticClassifications2(fileName: string, span: ts.TextSpan): ts.Classifications {
             return unwrapJSONCallResult(this.shim.getSemanticClassifications2(fileName, span.start, span.length));
         }
         getCompletionsAtPosition(fileName: string, position: number): ts.CompletionInfo {

src/server/client.ts

@@ -533,11 +533,11 @@ module ts.server {
             throw new Error("Not Implemented Yet.");
         }
 
-        getSyntacticClassifications2(fileName: string, span: TextSpan): number[] {
+        getSyntacticClassifications2(fileName: string, span: TextSpan): Classifications {
             throw new Error("Not Implemented Yet.");
         }
 
-        getSemanticClassifications2(fileName: string, span: TextSpan): number[] {
+        getSemanticClassifications2(fileName: string, span: TextSpan): Classifications {
             throw new Error("Not Implemented Yet.");
         }
 

src/services/services.ts

@@ -972,12 +972,19 @@ module ts {
         getSemanticDiagnostics(fileName: string): Diagnostic[];
         getCompilerOptionsDiagnostics(): Diagnostic[];
 
+        /** 
+         * @deprecated Use getSyntacticClassifications2 instead.
+         */
         getSyntacticClassifications(fileName: string, span: TextSpan): ClassifiedSpan[];
+
+        /** 
+         * @deprecated Use getSemanticClassifications2 instead.
+         */
         getSemanticClassifications(fileName: string, span: TextSpan): ClassifiedSpan[];
 
         // Encoded as triples of [start, length, ClassificationType].  
-        getSyntacticClassifications2(fileName: string, span: TextSpan): number[];
-        getSemanticClassifications2(fileName: string, span: TextSpan): number[];
+        getSyntacticClassifications2(fileName: string, span: TextSpan): Classifications;
+        getSemanticClassifications2(fileName: string, span: TextSpan): Classifications;
 
         getCompletionsAtPosition(fileName: string, position: number): CompletionInfo;
         getCompletionEntryDetails(fileName: string, position: number, entryName: string): CompletionEntryDetails;
@@ -1022,6 +1029,11 @@ module ts {
         dispose(): void;
     }
 
+    export interface Classifications {
+        spans: number[],
+        endOfLineState: EndOfLineState
+    }
+
     export interface ClassifiedSpan {
         textSpan: TextSpan;
         classificationType: string; // ClassificationTypeNames
@@ -1265,7 +1277,7 @@ module ts {
     }
 
     export const enum EndOfLineState {
-        Start,
+        None,
         InMultiLineCommentTrivia,
         InSingleQuoteStringLiteral,
         InDoubleQuoteStringLiteral,
@@ -1315,8 +1327,10 @@ module ts {
          *                                  classifications which may be incorrectly categorized will be given
          *                                  back as Identifiers in order to allow the syntactic classifier to
          *                                  subsume the classification.
+         * @deprecated Use getLexicalClassifications instead.
          */
         getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult;
+        getLexicalClassifications2(text: string, endOfLineState: EndOfLineState, syntacticClassifierAbsent: boolean): Classifications;
     }
 
     /**
@@ -1501,15 +1515,17 @@ module ts {
         numericLiteral = 4,
         operator = 5,
         stringLiteral = 6,
-        whiteSpace = 7,
-        text = 8,
-        punctuation = 9,
-        className = 10,
-        enumName = 11,
-        interfaceName = 12,
-        moduleName = 13,
-        typeParameterName = 14,
-        typeAlias = 15,
+        regularExpressionLiteral = 7,
+        whiteSpace = 8,
+        text = 9,
+        punctuation = 10,
+        className = 11,
+        enumName = 12,
+        interfaceName = 13,
+        moduleName = 14,
+        typeParameterName = 15,
+        typeAlias = 16,
+        parameterName = 17
     }
 
     /// Language Service
@@ -5830,7 +5846,7 @@ module ts {
             return convertClassifications(getSemanticClassifications2(fileName, span));
         }
 
-        function getSemanticClassifications2(fileName: string, span: TextSpan): number[] {
+        function getSemanticClassifications2(fileName: string, span: TextSpan): Classifications {
             synchronizeHostData();
 
             let sourceFile = getValidSourceFile(fileName);
@@ -5839,7 +5855,7 @@ module ts {
             let result: number[] = [];
             processNode(sourceFile);
 
-            return result;
+            return { spans: result, endOfLineState: EndOfLineState.None };
 
             function pushClassification(start: number, length: number, type: ClassificationType) {
                 result.push(start);
@@ -5927,8 +5943,9 @@ module ts {
             }
         }
 
-        function convertClassifications(dense: number[]): ClassifiedSpan[] {
-            Debug.assert(dense.length % 3 === 0);
+        function convertClassifications(classifications: Classifications): ClassifiedSpan[] {
+            Debug.assert(classifications.spans.length % 3 === 0);
+            let dense = classifications.spans;
             let result: ClassifiedSpan[] = [];
             for (let i = 0, n = dense.length; i < n; i += 3) {
                 result.push({
@@ -5944,7 +5961,7 @@ module ts {
             return convertClassifications(getSyntacticClassifications2(fileName, span));
         }
 
-        function getSyntacticClassifications2(fileName: string, span: TextSpan): number[] {
+        function getSyntacticClassifications2(fileName: string, span: TextSpan): Classifications {
             // doesn't use compiler - no need to synchronize with host
             let sourceFile = syntaxTreeCache.getCurrentSourceFile(fileName);
 
@@ -5955,7 +5972,7 @@ module ts {
             let result: number[] = [];
             processElement(sourceFile);
 
-            return result;
+            return { spans: result, endOfLineState: EndOfLineState.None };
 
             function pushClassification(start: number, length: number, type: ClassificationType) {
                 result.push(start);
@@ -6603,10 +6620,67 @@ module ts {
             // if there are more cases we want the classifier to be better at.
             return true;
         }
-        
+
+        function convertClassifications(classifications: Classifications, text: string): ClassificationResult {
+            var entries: ClassificationInfo[] = [];
+            let dense = classifications.spans;
+            let lastEnd = 0;
+
+            for (let i = 0, n = dense.length; i < n; i += 3) {
+                let start = dense[i];
+                let length = dense[i + 1];
+                let type = <ClassificationType>dense[i + 2];
+
+                // Make a whitespace entry between the last item and this one.
+                if (lastEnd >= 0) {
+                    let whitespaceLength = start - lastEnd;
+                    if (whitespaceLength > 0) {
+                        entries.push({ length: whitespaceLength, classification: TokenClass.Whitespace });
+                    }
+                }
+
+                entries.push({ length, classification: convertClassification(type) });
+                lastEnd = start + length;
+            }
+
+            let whitespaceLength = text.length - lastEnd;
+            if (whitespaceLength > 0) {
+                entries.push({ length: whitespaceLength, classification: TokenClass.Whitespace });
+            }
+
+            return { entries, finalLexState: classifications.endOfLineState };
+        }
+
+        function convertClassification(type: ClassificationType): TokenClass {
+            switch (type) {
+                case ClassificationType.comment: return TokenClass.Comment;
+                case ClassificationType.keyword: return TokenClass.Keyword;
+                case ClassificationType.numericLiteral: return TokenClass.NumberLiteral;
+                case ClassificationType.operator: return TokenClass.Operator;
+                case ClassificationType.stringLiteral: return TokenClass.StringLiteral;
+                case ClassificationType.whiteSpace: return TokenClass.Whitespace;
+                case ClassificationType.punctuation: return TokenClass.Punctuation;
+                case ClassificationType.identifier: 
+                case ClassificationType.className:
+                case ClassificationType.enumName:
+                case ClassificationType.interfaceName:
+                case ClassificationType.moduleName:
+                case ClassificationType.typeParameterName:
+                case ClassificationType.typeAlias:
+                case ClassificationType.text:
+                case ClassificationType.parameterName:
+                default:
+                    return TokenClass.Identifier;
+            }
+        }
+
+        function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult {
+            return convertClassifications(getLexicalClassifications2(text, lexState, syntacticClassifierAbsent), text);
+        }
+                
         // If there is a syntactic classifier ('syntacticClassifierAbsent' is false),
         // we will be more conservative in order to avoid conflicting with the syntactic classifier.
-        function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult {
+        function getLexicalClassifications2(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): Classifications {
             let offset = 0;
             let token = SyntaxKind.Unknown;
             let lastNonTriviaToken = SyntaxKind.Unknown;
@@ -6649,9 +6723,9 @@ module ts {
 
             scanner.setText(text);
 
-            let result: ClassificationResult = {
-                finalLexState: EndOfLineState.Start,
-                entries: []
+            let result: Classifications = {
+                endOfLineState: EndOfLineState.None,
+                spans: []
             };
 
             // We can run into an unfortunate interaction between the lexical and syntactic classifier
@@ -6764,7 +6838,7 @@ module ts {
                 let start = scanner.getTokenPos();
                 let end = scanner.getTextPos();
 
-                addResult(end - start, classFromKind(token));
+                addResult(start, end, classFromKind(token));
 
                 if (end >= text.length) {
                     if (token === SyntaxKind.StringLiteral) {
@@ -6781,7 +6855,7 @@ module ts {
                             // If we have an odd number of backslashes, then the multiline string is unclosed
                             if (numBackslashes & 1) {
                                 let quoteChar = tokenText.charCodeAt(0);
-                                result.finalLexState = quoteChar === CharacterCodes.doubleQuote
+                                result.endOfLineState = quoteChar === CharacterCodes.doubleQuote
                                     ? EndOfLineState.InDoubleQuoteStringLiteral
                                     : EndOfLineState.InSingleQuoteStringLiteral;
                             }
@@ -6790,16 +6864,16 @@ module ts {
                     else if (token === SyntaxKind.MultiLineCommentTrivia) {
                         // Check to see if the multiline comment was unclosed.
                         if (scanner.isUnterminated()) {
-                            result.finalLexState = EndOfLineState.InMultiLineCommentTrivia;
+                            result.endOfLineState = EndOfLineState.InMultiLineCommentTrivia;
                         }
                     }
                     else if (isTemplateLiteralKind(token)) {
                         if (scanner.isUnterminated()) {
                             if (token === SyntaxKind.TemplateTail) {
-                                result.finalLexState = EndOfLineState.InTemplateMiddleOrTail;
+                                result.endOfLineState = EndOfLineState.InTemplateMiddleOrTail;
                             }
                             else if (token === SyntaxKind.NoSubstitutionTemplateLiteral) {
-                                result.finalLexState = EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate;
+                                result.endOfLineState = EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate;
                             }
                             else {
                                 Debug.fail("Only 'NoSubstitutionTemplateLiteral's and 'TemplateTail's can be unterminated; got SyntaxKind #" + token);
@@ -6807,20 +6881,34 @@ module ts {
                         }
                     }
                     else if (templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) {
-                        result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition;
+                        result.endOfLineState = EndOfLineState.InTemplateSubstitutionPosition;
                     }
                 }
             }
 
-            function addResult(length: number, classification: TokenClass): void {
-                if (length > 0) {
-                    // If this is the first classification we're adding to the list, then remove any 
-                    // offset we have if we were continuing a construct from the previous line.
-                    if (result.entries.length === 0) {
-                        length -= offset;
-                    }
+            function addResult(start: number, end: number, classification: ClassificationType): void {
+                if (classification === ClassificationType.whiteSpace) {
+                    // Don't bother with whitespace classifications.  They're not needed.
+                    return;
+                }
 
-                    result.entries.push({ length: length, classification: classification });
+                if (start === 0 && offset > 0) {
+                    // We're classifying the first token, and this was a case where we prepended 
+                    // text.  We should consider the start of this token to be at the start of 
+                    // the original text.
+                    start += offset;
+                }
+
+                // All our tokens are in relation to the augmented text.  Move them back to be
+                // relative to the original text.
+                start -= offset;
+                end -= offset;
+                let length = end - start;
+
+                if (length > 0) {
+                    result.spans.push(start);
+                    result.spans.push(length);
+                    result.spans.push(classification);
                 }
             }
         }
@@ -6887,41 +6975,44 @@ module ts {
             return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword;
         }
 
-        function classFromKind(token: SyntaxKind) {
+        function classFromKind(token: SyntaxKind): ClassificationType {
             if (isKeyword(token)) {
-                return TokenClass.Keyword;
+                return ClassificationType.keyword;
             }
             else if (isBinaryExpressionOperatorToken(token) || isPrefixUnaryExpressionOperatorToken(token)) {
-                return TokenClass.Operator;
+                return ClassificationType.operator;
             }
             else if (token >= SyntaxKind.FirstPunctuation && token <= SyntaxKind.LastPunctuation) {
-                return TokenClass.Punctuation;
+                return ClassificationType.punctuation;
             }
 
             switch (token) {
                 case SyntaxKind.NumericLiteral:
-                    return TokenClass.NumberLiteral;
+                    return ClassificationType.numericLiteral;
                 case SyntaxKind.StringLiteral:
-                    return TokenClass.StringLiteral;
+                    return ClassificationType.stringLiteral;
                 case SyntaxKind.RegularExpressionLiteral:
-                    return TokenClass.RegExpLiteral;
+                    return ClassificationType.regularExpressionLiteral;
                 case SyntaxKind.ConflictMarkerTrivia:
                 case SyntaxKind.MultiLineCommentTrivia:
                 case SyntaxKind.SingleLineCommentTrivia:
-                    return TokenClass.Comment;
+                    return ClassificationType.comment;
                 case SyntaxKind.WhitespaceTrivia:
                 case SyntaxKind.NewLineTrivia:
-                    return TokenClass.Whitespace;
+                    return ClassificationType.whiteSpace;
                 case SyntaxKind.Identifier:
                 default:
                     if (isTemplateLiteralKind(token)) {
-                        return TokenClass.StringLiteral;
+                        return ClassificationType.stringLiteral;
                     }
-                    return TokenClass.Identifier;
+                    return ClassificationType.identifier;
             }
         }
 
-        return { getClassificationsForLine };
+        return {
+            getClassificationsForLine,
+            getLexicalClassifications2
+        };
     }
 
     /// getDefaultLibraryFilePath

src/services/shims.ts

@@ -185,6 +185,7 @@ module ts {
     }
 
     export interface ClassifierShim extends Shim {
+        getLexicalClassifications2(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string;
         getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string;
     }
 
@@ -194,7 +195,9 @@ module ts {
     }
 
     function logInternalError(logger: Logger, err: Error) {
-        logger.log("*INTERNAL ERROR* - Exception in typescript services: " + err.message);
+        if (logger) {
+            logger.log("*INTERNAL ERROR* - Exception in typescript services: " + err.message);
+        }
     }
 
     class ScriptSnapshotShimAdapter implements IScriptSnapshot {
@@ -308,18 +311,25 @@ module ts {
     function simpleForwardCall(logger: Logger, actionDescription: string, action: () => any): any {
         return action();
 
-        logger.log(actionDescription);
+        if (logger) {
+            logger.log(actionDescription);
+        }
+
         var start = Date.now();
         var result = action();
         var end = Date.now();
-        logger.log(actionDescription + " completed in " + (end - start) + " msec");
-        if (typeof (result) === "string") {
-            var str = <string>result;
-            if (str.length > 128) {
-                str = str.substring(0, 128) + "...";
+
+        if (logger) {
+            logger.log(actionDescription + " completed in " + (end - start) + " msec");
+            if (typeof (result) === "string") {
+                var str = <string>result;
+                if (str.length > 128) {
+                    str = str.substring(0, 128) + "...";
+                }
+                logger.log("  result.length=" + str.length + ", result='" + JSON.stringify(str) + "'");
             }
-            logger.log("  result.length=" + str.length + ", result='" + JSON.stringify(str) + "'");
         }
+
         return result;
     }
 
@@ -447,8 +457,9 @@ module ts {
             return this.forwardJSONCall(
                 "getSyntacticClassifications('" + fileName + "', " + start + ", " + length + ")",
                 () => {
-                    var classifications = this.languageService.getSyntacticClassifications2(fileName, createTextSpan(start, length));
-                    return classifications.join(",");
+                    // directly serialize the spans out to a string.  This is much faster to decode
+                    // on the managed side versus a full JSON array.
+                    return convertClassifications(this.languageService.getSyntacticClassifications2(fileName, createTextSpan(start, length)));
                 });
         }
 
@@ -456,8 +467,9 @@ module ts {
             return this.forwardJSONCall(
                 "getSemanticClassifications('" + fileName + "', " + start + ", " + length + ")",
                 () => {
-                    var classifications = this.languageService.getSemanticClassifications2(fileName, createTextSpan(start, length));
-                    return classifications.join(",");
+                    // directly serialize the spans out to a string.  This is much faster to decode
+                    // on the managed side versus a full JSON array.
+                    return convertClassifications(this.languageService.getSemanticClassifications2(fileName, createTextSpan(start, length)));
                 });
         }
 
@@ -740,6 +752,10 @@ module ts {
         }
     }
 
+    function convertClassifications(classifications: Classifications): { spans: string, endOfLineState: EndOfLineState } {
+        return { spans: classifications.spans.join(","), endOfLineState: classifications.endOfLineState };
+    }
+
     class ClassifierShimObject extends ShimBase implements ClassifierShim {
         public classifier: Classifier;
 
@@ -748,6 +764,11 @@ module ts {
             this.classifier = createClassifier();
         }
 
+        public getLexicalClassifications2(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string {
+            return forwardJSONCall(/*logger:*/ undefined, "getLexicalClassifications2",
+                () => convertClassifications(this.classifier.getLexicalClassifications2(text, lexState, syntacticClassifierAbsent)));
+        }
+
         /// COLORIZATION
         public getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): string {
             var classification = this.classifier.getClassificationsForLine(text, lexState, classifyKeywordsInGenerics);

tests/cases/unittests/services/colorization.ts

@@ -66,8 +66,9 @@ describe('Colorization', function () {
 
     describe("test getClassifications", function () {
         it("Returns correct token classes", function () {
+            debugger;
             testLexicalClassification("var x: string = \"foo\"; //Hello",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 keyword("var"),
                 whitespace(" "),
                 identifier("x"),
@@ -81,7 +82,7 @@ describe('Colorization', function () {
 
         it("correctly classifies a comment after a divide operator", function () {
             testLexicalClassification("1 / 2 // comment",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 numberLiteral("1"),
                 whitespace(" "),
                 operator("/"),
@@ -91,7 +92,7 @@ describe('Colorization', function () {
 
         it("correctly classifies a literal after a divide operator", function () {
             testLexicalClassification("1 / 2, 3 / 4",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 numberLiteral("1"),
                 whitespace(" "),
                 operator("/"),
@@ -103,40 +104,41 @@ describe('Colorization', function () {
 
         it("correctly classifies a multi-line string with one backslash", function () {
             testLexicalClassification("'line1\\",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 stringLiteral("'line1\\"),
                 finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral));
         });
 
         it("correctly classifies a multi-line string with three backslashes", function () {
             testLexicalClassification("'line1\\\\\\",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 stringLiteral("'line1\\\\\\"),
                 finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral));
         });
 
         it("correctly classifies an unterminated single-line string with no backslashes", function () {
             testLexicalClassification("'line1",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 stringLiteral("'line1"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("correctly classifies an unterminated single-line string with two backslashes", function () {
             testLexicalClassification("'line1\\\\",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 stringLiteral("'line1\\\\"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("correctly classifies an unterminated single-line string with four backslashes", function () {
             testLexicalClassification("'line1\\\\\\\\",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 stringLiteral("'line1\\\\\\\\"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("correctly classifies the continuing line of a multi-line string ending in one backslash", function () {
+            debugger;
             testLexicalClassification("\\",
                 ts.EndOfLineState.InDoubleQuoteStringLiteral,
                 stringLiteral("\\"),
@@ -154,33 +156,33 @@ describe('Colorization', function () {
             testLexicalClassification("  ",
                 ts.EndOfLineState.InDoubleQuoteStringLiteral,
                 stringLiteral("  "),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("correctly classifies the last line of an unterminated multi-line string ending in two backslashes", function () {
             testLexicalClassification("\\\\",
                 ts.EndOfLineState.InDoubleQuoteStringLiteral,
                 stringLiteral("\\\\"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("correctly classifies the last line of an unterminated multi-line string ending in four backslashes", function () {
             testLexicalClassification("\\\\\\\\",
                 ts.EndOfLineState.InDoubleQuoteStringLiteral,
                 stringLiteral("\\\\\\\\"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("correctly classifies the last line of a multi-line string", function () {
             testLexicalClassification("'",
                 ts.EndOfLineState.InSingleQuoteStringLiteral,
                 stringLiteral("'"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("correctly classifies an unterminated multiline comment", function () {
             testLexicalClassification("/*",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 comment("/*"),
                 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia));
         });
@@ -189,7 +191,7 @@ describe('Colorization', function () {
             testLexicalClassification("   */     ",
                 ts.EndOfLineState.InMultiLineCommentTrivia,
                 comment("   */"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("correctly classifies the continuation of a multiline comment", function () {
@@ -201,33 +203,33 @@ describe('Colorization', function () {
 
         it("correctly classifies an unterminated multiline comment on a line ending in '/*/'", function () {
             testLexicalClassification("   /*/",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 comment("/*/"),
                 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia));
         });
 
         it("correctly classifies an unterminated multiline comment with trailing space", function () {
             testLexicalClassification("/* ",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 comment("/* "),
                 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia));
         });
 
         it("correctly classifies a keyword after a dot", function () {
             testLexicalClassification("a.var",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 identifier("var"));
         });
 
         it("correctly classifies a string literal after a dot", function () {
             testLexicalClassification("a.\"var\"",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 stringLiteral("\"var\""));
         });
 
         it("correctly classifies a keyword after a dot separated by comment trivia", function () {
             testLexicalClassification("a./*hello world*/ var",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 identifier("a"),
                 punctuation("."),
                 comment("/*hello world*/"),
@@ -236,41 +238,41 @@ describe('Colorization', function () {
 
         it("classifies a property access with whitespace around the dot", function () {
             testLexicalClassification("   x  .\tfoo ()",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 identifier("x"),
                 identifier("foo"));
         });
 
         it("classifies a keyword after a dot on previous line", function () {
             testLexicalClassification("var",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 keyword("var"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("classifies multiple keywords properly", function () {
             testLexicalClassification("public static",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 keyword("public"),
                 keyword("static"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
 
             testLexicalClassification("public var",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 keyword("public"),
                 identifier("var"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("classifies a single line no substitution template string correctly", () => {
             testLexicalClassification("`number number public string`",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 stringLiteral("`number number public string`"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
         it("classifies substitution parts of a template string correctly", () => {
             testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 stringLiteral("`number '${"),
                 numberLiteral("1"),
                 operator("+"),
@@ -278,11 +280,11 @@ describe('Colorization', function () {
                 stringLiteral("}' string '${"),
                 stringLiteral("'hello'"),
                 stringLiteral("}'`"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
         it("classifies an unterminated no substitution template string correctly", () => {
             testLexicalClassification("`hello world",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 stringLiteral("`hello world"),
                 finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate));
         });
@@ -308,7 +310,7 @@ describe('Colorization', function () {
             testLexicalClassification("...`",
                 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
                 stringLiteral("...`"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
         it("classifies the substitution parts and middle/tail of a multiline template string", () => {
             testLexicalClassification("${ 1 + 1 }...`",
@@ -318,7 +320,7 @@ describe('Colorization', function () {
                 operator("+"),
                 numberLiteral("1"),
                 stringLiteral("}...`"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
         it("classifies a template middle and propagates the end of line state",() => {
             testLexicalClassification("${ 1 + 1 }...`",
@@ -328,7 +330,7 @@ describe('Colorization', function () {
                 operator("+"),
                 numberLiteral("1"),
                 stringLiteral("}...`"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
         it("classifies substitution expressions with curly braces appropriately", () => {
             var pos = 0;
@@ -349,7 +351,7 @@ describe('Colorization', function () {
                 stringLiteral(track(" ", "`1`"), pos),
                 punctuation(track(" ", "}"), pos),
                 stringLiteral(track(" ", "}...`"), pos),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
 
             // Adjusts 'pos' by accounting for the length of each portion of the string,
             // but only return the last given string
@@ -364,22 +366,22 @@ describe('Colorization', function () {
 
         it("classifies partially written generics correctly.", function () {
             testLexicalClassification("Foo<number",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 identifier("Foo"),
                 operator("<"),
                 identifier("number"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
 
             // Looks like a cast, should get classified as a keyword.
             testLexicalClassification("<number",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 operator("<"),
                 keyword("number"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
 
             // handle nesting properly.
             testLexicalClassification("Foo<Foo,Foo<number",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 identifier("Foo"),
                 operator("<"),
                 identifier("Foo"),
@@ -387,7 +389,7 @@ describe('Colorization', function () {
                 identifier("Foo"),
                 operator("<"),
                 identifier("number"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("LexicallyClassifiesConflictTokens", () => {
@@ -400,7 +402,7 @@ describe('Colorization', function () {
     v = 2;\r\n\
 >>>>>>> Branch - a\r\n\
 }",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 keyword("class"),
                 identifier("C"),
                 punctuation("{"),
@@ -412,7 +414,7 @@ describe('Colorization', function () {
                 comment("=======\r\n    v = 2;\r\n"),
                 comment(">>>>>>> Branch - a"),
                 punctuation("}"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
 
             testLexicalClassification(
 "<<<<<<< HEAD\r\n\
@@ -420,7 +422,7 @@ class C { }\r\n\
 =======\r\n\
 class D { }\r\n\
 >>>>>>> Branch - a\r\n",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 comment("<<<<<<< HEAD"),
                 keyword("class"),
                 identifier("C"),
@@ -428,12 +430,12 @@ class D { }\r\n\
                 punctuation("}"),
                 comment("=======\r\nclass D { }\r\n"),
                 comment(">>>>>>> Branch - a"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
 
         it("'of' keyword", function () {
             testLexicalClassification("for (var of of of) { }",
-                ts.EndOfLineState.Start,
+                ts.EndOfLineState.None,
                 keyword("for"),
                 punctuation("("),
                 keyword("var"),
@@ -443,7 +445,7 @@ class D { }\r\n\
                 punctuation(")"),
                 punctuation("{"),
                 punctuation("}"),
-                finalEndOfLineState(ts.EndOfLineState.Start));
+                finalEndOfLineState(ts.EndOfLineState.None));
         });
     });
 });