From 089fd4dd8f853e9d0164fa010796ca451bfe54b3 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 30 Apr 2015 14:26:46 -0700 Subject: [PATCH] Add a common, dense, format for classification operations to lower cost of processing on the host side. We now just return an array of triples to represent classified results. The triple contains: 1) the start of the classification. 2) the length of the classification. 3) the type of the clasification. We also encode this into a comma separated string when passing over to the managed side (as opposed to an JSON array). That way we don't pay such a high JSON parsing cost. Instead, we can just do a string.split(",") on the encoded triples and process each element ourselves. --- src/harness/harnessLanguageService.ts | 7 +- src/server/client.ts | 4 +- src/services/services.ts | 189 +++++++++++++----- src/services/shims.ts | 45 +++-- .../cases/unittests/services/colorization.ts | 102 +++++----- 5 files changed, 232 insertions(+), 115 deletions(-) diff --git a/src/harness/harnessLanguageService.ts b/src/harness/harnessLanguageService.ts index 052a3900ad6..289d181face 100644 --- a/src/harness/harnessLanguageService.ts +++ b/src/harness/harnessLanguageService.ts @@ -235,6 +235,9 @@ module Harness.LanguageService { class ClassifierShimProxy implements ts.Classifier { constructor(private shim: ts.ClassifierShim) { } + getLexicalClassifications2(text: string, lexState: ts.EndOfLineState, classifyKeywordsInGenerics?: boolean): ts.Classifications { + throw new Error("NYI"); + } getClassificationsForLine(text: string, lexState: ts.EndOfLineState, classifyKeywordsInGenerics?: boolean): ts.ClassificationResult { var result = this.shim.getClassificationsForLine(text, lexState, classifyKeywordsInGenerics).split('\n'); var entries: ts.ClassificationInfo[] = []; @@ -300,10 +303,10 @@ module Harness.LanguageService { getSemanticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] { return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length)); } - getSyntacticClassifications2(fileName: string, span: ts.TextSpan): number[] { + getSyntacticClassifications2(fileName: string, span: ts.TextSpan): ts.Classifications { return unwrapJSONCallResult(this.shim.getSyntacticClassifications2(fileName, span.start, span.length)); } - getSemanticClassifications2(fileName: string, span: ts.TextSpan): number[] { + getSemanticClassifications2(fileName: string, span: ts.TextSpan): ts.Classifications { return unwrapJSONCallResult(this.shim.getSemanticClassifications2(fileName, span.start, span.length)); } getCompletionsAtPosition(fileName: string, position: number): ts.CompletionInfo { diff --git a/src/server/client.ts b/src/server/client.ts index addf05f53d5..967cf6d9a1f 100644 --- a/src/server/client.ts +++ b/src/server/client.ts @@ -533,11 +533,11 @@ module ts.server { throw new Error("Not Implemented Yet."); } - getSyntacticClassifications2(fileName: string, span: TextSpan): number[] { + getSyntacticClassifications2(fileName: string, span: TextSpan): Classifications { throw new Error("Not Implemented Yet."); } - getSemanticClassifications2(fileName: string, span: TextSpan): number[] { + getSemanticClassifications2(fileName: string, span: TextSpan): Classifications { throw new Error("Not Implemented Yet."); } diff --git a/src/services/services.ts b/src/services/services.ts index 85af0a374d6..7fbbcda45d5 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -972,12 +972,19 @@ module ts { getSemanticDiagnostics(fileName: string): Diagnostic[]; getCompilerOptionsDiagnostics(): Diagnostic[]; + /** + * @deprecated Use getSyntacticClassifications2 instead. + */ getSyntacticClassifications(fileName: string, span: TextSpan): ClassifiedSpan[]; + + /** + * @deprecated Use getSemanticClassifications2 instead. + */ getSemanticClassifications(fileName: string, span: TextSpan): ClassifiedSpan[]; // Encoded as triples of [start, length, ClassificationType]. - getSyntacticClassifications2(fileName: string, span: TextSpan): number[]; - getSemanticClassifications2(fileName: string, span: TextSpan): number[]; + getSyntacticClassifications2(fileName: string, span: TextSpan): Classifications; + getSemanticClassifications2(fileName: string, span: TextSpan): Classifications; getCompletionsAtPosition(fileName: string, position: number): CompletionInfo; getCompletionEntryDetails(fileName: string, position: number, entryName: string): CompletionEntryDetails; @@ -1022,6 +1029,11 @@ module ts { dispose(): void; } + export interface Classifications { + spans: number[], + endOfLineState: EndOfLineState + } + export interface ClassifiedSpan { textSpan: TextSpan; classificationType: string; // ClassificationTypeNames @@ -1265,7 +1277,7 @@ module ts { } export const enum EndOfLineState { - Start, + None, InMultiLineCommentTrivia, InSingleQuoteStringLiteral, InDoubleQuoteStringLiteral, @@ -1315,8 +1327,10 @@ module ts { * classifications which may be incorrectly categorized will be given * back as Identifiers in order to allow the syntactic classifier to * subsume the classification. + * @deprecated Use getLexicalClassifications instead. */ getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; + getLexicalClassifications2(text: string, endOfLineState: EndOfLineState, syntacticClassifierAbsent: boolean): Classifications; } /** @@ -1501,15 +1515,17 @@ module ts { numericLiteral = 4, operator = 5, stringLiteral = 6, - whiteSpace = 7, - text = 8, - punctuation = 9, - className = 10, - enumName = 11, - interfaceName = 12, - moduleName = 13, - typeParameterName = 14, - typeAlias = 15, + regularExpressionLiteral = 7, + whiteSpace = 8, + text = 9, + punctuation = 10, + className = 11, + enumName = 12, + interfaceName = 13, + moduleName = 14, + typeParameterName = 15, + typeAlias = 16, + parameterName = 17 } /// Language Service @@ -5830,7 +5846,7 @@ module ts { return convertClassifications(getSemanticClassifications2(fileName, span)); } - function getSemanticClassifications2(fileName: string, span: TextSpan): number[] { + function getSemanticClassifications2(fileName: string, span: TextSpan): Classifications { synchronizeHostData(); let sourceFile = getValidSourceFile(fileName); @@ -5839,7 +5855,7 @@ module ts { let result: number[] = []; processNode(sourceFile); - return result; + return { spans: result, endOfLineState: EndOfLineState.None }; function pushClassification(start: number, length: number, type: ClassificationType) { result.push(start); @@ -5927,8 +5943,9 @@ module ts { } } - function convertClassifications(dense: number[]): ClassifiedSpan[] { - Debug.assert(dense.length % 3 === 0); + function convertClassifications(classifications: Classifications): ClassifiedSpan[] { + Debug.assert(classifications.spans.length % 3 === 0); + let dense = classifications.spans; let result: ClassifiedSpan[] = []; for (let i = 0, n = dense.length; i < n; i += 3) { result.push({ @@ -5944,7 +5961,7 @@ module ts { return convertClassifications(getSyntacticClassifications2(fileName, span)); } - function getSyntacticClassifications2(fileName: string, span: TextSpan): number[] { + function getSyntacticClassifications2(fileName: string, span: TextSpan): Classifications { // doesn't use compiler - no need to synchronize with host let sourceFile = syntaxTreeCache.getCurrentSourceFile(fileName); @@ -5955,7 +5972,7 @@ module ts { let result: number[] = []; processElement(sourceFile); - return result; + return { spans: result, endOfLineState: EndOfLineState.None }; function pushClassification(start: number, length: number, type: ClassificationType) { result.push(start); @@ -6603,10 +6620,67 @@ module ts { // if there are more cases we want the classifier to be better at. return true; } - + + function convertClassifications(classifications: Classifications, text: string): ClassificationResult { + var entries: ClassificationInfo[] = []; + let dense = classifications.spans; + let lastEnd = 0; + + for (let i = 0, n = dense.length; i < n; i += 3) { + let start = dense[i]; + let length = dense[i + 1]; + let type = dense[i + 2]; + + // Make a whitespace entry between the last item and this one. + if (lastEnd >= 0) { + let whitespaceLength = start - lastEnd; + if (whitespaceLength > 0) { + entries.push({ length: whitespaceLength, classification: TokenClass.Whitespace }); + } + } + + entries.push({ length, classification: convertClassification(type) }); + lastEnd = start + length; + } + + let whitespaceLength = text.length - lastEnd; + if (whitespaceLength > 0) { + entries.push({ length: whitespaceLength, classification: TokenClass.Whitespace }); + } + + return { entries, finalLexState: classifications.endOfLineState }; + } + + function convertClassification(type: ClassificationType): TokenClass { + switch (type) { + case ClassificationType.comment: return TokenClass.Comment; + case ClassificationType.keyword: return TokenClass.Keyword; + case ClassificationType.numericLiteral: return TokenClass.NumberLiteral; + case ClassificationType.operator: return TokenClass.Operator; + case ClassificationType.stringLiteral: return TokenClass.StringLiteral; + case ClassificationType.whiteSpace: return TokenClass.Whitespace; + case ClassificationType.punctuation: return TokenClass.Punctuation; + case ClassificationType.identifier: + case ClassificationType.className: + case ClassificationType.enumName: + case ClassificationType.interfaceName: + case ClassificationType.moduleName: + case ClassificationType.typeParameterName: + case ClassificationType.typeAlias: + case ClassificationType.text: + case ClassificationType.parameterName: + default: + return TokenClass.Identifier; + } + } + + function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult { + return convertClassifications(getLexicalClassifications2(text, lexState, syntacticClassifierAbsent), text); + } + // If there is a syntactic classifier ('syntacticClassifierAbsent' is false), // we will be more conservative in order to avoid conflicting with the syntactic classifier. - function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult { + function getLexicalClassifications2(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): Classifications { let offset = 0; let token = SyntaxKind.Unknown; let lastNonTriviaToken = SyntaxKind.Unknown; @@ -6649,9 +6723,9 @@ module ts { scanner.setText(text); - let result: ClassificationResult = { - finalLexState: EndOfLineState.Start, - entries: [] + let result: Classifications = { + endOfLineState: EndOfLineState.None, + spans: [] }; // We can run into an unfortunate interaction between the lexical and syntactic classifier @@ -6764,7 +6838,7 @@ module ts { let start = scanner.getTokenPos(); let end = scanner.getTextPos(); - addResult(end - start, classFromKind(token)); + addResult(start, end, classFromKind(token)); if (end >= text.length) { if (token === SyntaxKind.StringLiteral) { @@ -6781,7 +6855,7 @@ module ts { // If we have an odd number of backslashes, then the multiline string is unclosed if (numBackslashes & 1) { let quoteChar = tokenText.charCodeAt(0); - result.finalLexState = quoteChar === CharacterCodes.doubleQuote + result.endOfLineState = quoteChar === CharacterCodes.doubleQuote ? EndOfLineState.InDoubleQuoteStringLiteral : EndOfLineState.InSingleQuoteStringLiteral; } @@ -6790,16 +6864,16 @@ module ts { else if (token === SyntaxKind.MultiLineCommentTrivia) { // Check to see if the multiline comment was unclosed. if (scanner.isUnterminated()) { - result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; + result.endOfLineState = EndOfLineState.InMultiLineCommentTrivia; } } else if (isTemplateLiteralKind(token)) { if (scanner.isUnterminated()) { if (token === SyntaxKind.TemplateTail) { - result.finalLexState = EndOfLineState.InTemplateMiddleOrTail; + result.endOfLineState = EndOfLineState.InTemplateMiddleOrTail; } else if (token === SyntaxKind.NoSubstitutionTemplateLiteral) { - result.finalLexState = EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate; + result.endOfLineState = EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate; } else { Debug.fail("Only 'NoSubstitutionTemplateLiteral's and 'TemplateTail's can be unterminated; got SyntaxKind #" + token); @@ -6807,20 +6881,34 @@ module ts { } } else if (templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { - result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition; + result.endOfLineState = EndOfLineState.InTemplateSubstitutionPosition; } } } - function addResult(length: number, classification: TokenClass): void { - if (length > 0) { - // If this is the first classification we're adding to the list, then remove any - // offset we have if we were continuing a construct from the previous line. - if (result.entries.length === 0) { - length -= offset; - } + function addResult(start: number, end: number, classification: ClassificationType): void { + if (classification === ClassificationType.whiteSpace) { + // Don't bother with whitespace classifications. They're not needed. + return; + } - result.entries.push({ length: length, classification: classification }); + if (start === 0 && offset > 0) { + // We're classifying the first token, and this was a case where we prepended + // text. We should consider the start of this token to be at the start of + // the original text. + start += offset; + } + + // All our tokens are in relation to the augmented text. Move them back to be + // relative to the original text. + start -= offset; + end -= offset; + let length = end - start; + + if (length > 0) { + result.spans.push(start); + result.spans.push(length); + result.spans.push(classification); } } } @@ -6887,41 +6975,44 @@ module ts { return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword; } - function classFromKind(token: SyntaxKind) { + function classFromKind(token: SyntaxKind): ClassificationType { if (isKeyword(token)) { - return TokenClass.Keyword; + return ClassificationType.keyword; } else if (isBinaryExpressionOperatorToken(token) || isPrefixUnaryExpressionOperatorToken(token)) { - return TokenClass.Operator; + return ClassificationType.operator; } else if (token >= SyntaxKind.FirstPunctuation && token <= SyntaxKind.LastPunctuation) { - return TokenClass.Punctuation; + return ClassificationType.punctuation; } switch (token) { case SyntaxKind.NumericLiteral: - return TokenClass.NumberLiteral; + return ClassificationType.numericLiteral; case SyntaxKind.StringLiteral: - return TokenClass.StringLiteral; + return ClassificationType.stringLiteral; case SyntaxKind.RegularExpressionLiteral: - return TokenClass.RegExpLiteral; + return ClassificationType.regularExpressionLiteral; case SyntaxKind.ConflictMarkerTrivia: case SyntaxKind.MultiLineCommentTrivia: case SyntaxKind.SingleLineCommentTrivia: - return TokenClass.Comment; + return ClassificationType.comment; case SyntaxKind.WhitespaceTrivia: case SyntaxKind.NewLineTrivia: - return TokenClass.Whitespace; + return ClassificationType.whiteSpace; case SyntaxKind.Identifier: default: if (isTemplateLiteralKind(token)) { - return TokenClass.StringLiteral; + return ClassificationType.stringLiteral; } - return TokenClass.Identifier; + return ClassificationType.identifier; } } - return { getClassificationsForLine }; + return { + getClassificationsForLine, + getLexicalClassifications2 + }; } /// getDefaultLibraryFilePath diff --git a/src/services/shims.ts b/src/services/shims.ts index 835383d91f2..55b89aba91f 100644 --- a/src/services/shims.ts +++ b/src/services/shims.ts @@ -185,6 +185,7 @@ module ts { } export interface ClassifierShim extends Shim { + getLexicalClassifications2(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string; getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string; } @@ -194,7 +195,9 @@ module ts { } function logInternalError(logger: Logger, err: Error) { - logger.log("*INTERNAL ERROR* - Exception in typescript services: " + err.message); + if (logger) { + logger.log("*INTERNAL ERROR* - Exception in typescript services: " + err.message); + } } class ScriptSnapshotShimAdapter implements IScriptSnapshot { @@ -308,18 +311,25 @@ module ts { function simpleForwardCall(logger: Logger, actionDescription: string, action: () => any): any { return action(); - logger.log(actionDescription); + if (logger) { + logger.log(actionDescription); + } + var start = Date.now(); var result = action(); var end = Date.now(); - logger.log(actionDescription + " completed in " + (end - start) + " msec"); - if (typeof (result) === "string") { - var str = result; - if (str.length > 128) { - str = str.substring(0, 128) + "..."; + + if (logger) { + logger.log(actionDescription + " completed in " + (end - start) + " msec"); + if (typeof (result) === "string") { + var str = result; + if (str.length > 128) { + str = str.substring(0, 128) + "..."; + } + logger.log(" result.length=" + str.length + ", result='" + JSON.stringify(str) + "'"); } - logger.log(" result.length=" + str.length + ", result='" + JSON.stringify(str) + "'"); } + return result; } @@ -447,8 +457,9 @@ module ts { return this.forwardJSONCall( "getSyntacticClassifications('" + fileName + "', " + start + ", " + length + ")", () => { - var classifications = this.languageService.getSyntacticClassifications2(fileName, createTextSpan(start, length)); - return classifications.join(","); + // directly serialize the spans out to a string. This is much faster to decode + // on the managed side versus a full JSON array. + return convertClassifications(this.languageService.getSyntacticClassifications2(fileName, createTextSpan(start, length))); }); } @@ -456,8 +467,9 @@ module ts { return this.forwardJSONCall( "getSemanticClassifications('" + fileName + "', " + start + ", " + length + ")", () => { - var classifications = this.languageService.getSemanticClassifications2(fileName, createTextSpan(start, length)); - return classifications.join(","); + // directly serialize the spans out to a string. This is much faster to decode + // on the managed side versus a full JSON array. + return convertClassifications(this.languageService.getSemanticClassifications2(fileName, createTextSpan(start, length))); }); } @@ -740,6 +752,10 @@ module ts { } } + function convertClassifications(classifications: Classifications): { spans: string, endOfLineState: EndOfLineState } { + return { spans: classifications.spans.join(","), endOfLineState: classifications.endOfLineState }; + } + class ClassifierShimObject extends ShimBase implements ClassifierShim { public classifier: Classifier; @@ -748,6 +764,11 @@ module ts { this.classifier = createClassifier(); } + public getLexicalClassifications2(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string { + return forwardJSONCall(/*logger:*/ undefined, "getLexicalClassifications2", + () => convertClassifications(this.classifier.getLexicalClassifications2(text, lexState, syntacticClassifierAbsent))); + } + /// COLORIZATION public getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): string { var classification = this.classifier.getClassificationsForLine(text, lexState, classifyKeywordsInGenerics); diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index 6c685cdb174..4351f0ccb1d 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -66,8 +66,9 @@ describe('Colorization', function () { describe("test getClassifications", function () { it("Returns correct token classes", function () { + debugger; testLexicalClassification("var x: string = \"foo\"; //Hello", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, keyword("var"), whitespace(" "), identifier("x"), @@ -81,7 +82,7 @@ describe('Colorization', function () { it("correctly classifies a comment after a divide operator", function () { testLexicalClassification("1 / 2 // comment", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, numberLiteral("1"), whitespace(" "), operator("/"), @@ -91,7 +92,7 @@ describe('Colorization', function () { it("correctly classifies a literal after a divide operator", function () { testLexicalClassification("1 / 2, 3 / 4", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, numberLiteral("1"), whitespace(" "), operator("/"), @@ -103,40 +104,41 @@ describe('Colorization', function () { it("correctly classifies a multi-line string with one backslash", function () { testLexicalClassification("'line1\\", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, stringLiteral("'line1\\"), finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); }); it("correctly classifies a multi-line string with three backslashes", function () { testLexicalClassification("'line1\\\\\\", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, stringLiteral("'line1\\\\\\"), finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); }); it("correctly classifies an unterminated single-line string with no backslashes", function () { testLexicalClassification("'line1", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, stringLiteral("'line1"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("correctly classifies an unterminated single-line string with two backslashes", function () { testLexicalClassification("'line1\\\\", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, stringLiteral("'line1\\\\"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("correctly classifies an unterminated single-line string with four backslashes", function () { testLexicalClassification("'line1\\\\\\\\", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, stringLiteral("'line1\\\\\\\\"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("correctly classifies the continuing line of a multi-line string ending in one backslash", function () { + debugger; testLexicalClassification("\\", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral("\\"), @@ -154,33 +156,33 @@ describe('Colorization', function () { testLexicalClassification(" ", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral(" "), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("correctly classifies the last line of an unterminated multi-line string ending in two backslashes", function () { testLexicalClassification("\\\\", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral("\\\\"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("correctly classifies the last line of an unterminated multi-line string ending in four backslashes", function () { testLexicalClassification("\\\\\\\\", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral("\\\\\\\\"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("correctly classifies the last line of a multi-line string", function () { testLexicalClassification("'", ts.EndOfLineState.InSingleQuoteStringLiteral, stringLiteral("'"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("correctly classifies an unterminated multiline comment", function () { testLexicalClassification("/*", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, comment("/*"), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); @@ -189,7 +191,7 @@ describe('Colorization', function () { testLexicalClassification(" */ ", ts.EndOfLineState.InMultiLineCommentTrivia, comment(" */"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("correctly classifies the continuation of a multiline comment", function () { @@ -201,33 +203,33 @@ describe('Colorization', function () { it("correctly classifies an unterminated multiline comment on a line ending in '/*/'", function () { testLexicalClassification(" /*/", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, comment("/*/"), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); it("correctly classifies an unterminated multiline comment with trailing space", function () { testLexicalClassification("/* ", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, comment("/* "), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); it("correctly classifies a keyword after a dot", function () { testLexicalClassification("a.var", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, identifier("var")); }); it("correctly classifies a string literal after a dot", function () { testLexicalClassification("a.\"var\"", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, stringLiteral("\"var\"")); }); it("correctly classifies a keyword after a dot separated by comment trivia", function () { testLexicalClassification("a./*hello world*/ var", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, identifier("a"), punctuation("."), comment("/*hello world*/"), @@ -236,41 +238,41 @@ describe('Colorization', function () { it("classifies a property access with whitespace around the dot", function () { testLexicalClassification(" x .\tfoo ()", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, identifier("x"), identifier("foo")); }); it("classifies a keyword after a dot on previous line", function () { testLexicalClassification("var", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, keyword("var"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("classifies multiple keywords properly", function () { testLexicalClassification("public static", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, keyword("public"), keyword("static"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); testLexicalClassification("public var", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, keyword("public"), identifier("var"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("classifies a single line no substitution template string correctly", () => { testLexicalClassification("`number number public string`", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, stringLiteral("`number number public string`"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("classifies substitution parts of a template string correctly", () => { testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, stringLiteral("`number '${"), numberLiteral("1"), operator("+"), @@ -278,11 +280,11 @@ describe('Colorization', function () { stringLiteral("}' string '${"), stringLiteral("'hello'"), stringLiteral("}'`"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("classifies an unterminated no substitution template string correctly", () => { testLexicalClassification("`hello world", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, stringLiteral("`hello world"), finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); }); @@ -308,7 +310,7 @@ describe('Colorization', function () { testLexicalClassification("...`", ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, stringLiteral("...`"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("classifies the substitution parts and middle/tail of a multiline template string", () => { testLexicalClassification("${ 1 + 1 }...`", @@ -318,7 +320,7 @@ describe('Colorization', function () { operator("+"), numberLiteral("1"), stringLiteral("}...`"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("classifies a template middle and propagates the end of line state",() => { testLexicalClassification("${ 1 + 1 }...`", @@ -328,7 +330,7 @@ describe('Colorization', function () { operator("+"), numberLiteral("1"), stringLiteral("}...`"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("classifies substitution expressions with curly braces appropriately", () => { var pos = 0; @@ -349,7 +351,7 @@ describe('Colorization', function () { stringLiteral(track(" ", "`1`"), pos), punctuation(track(" ", "}"), pos), stringLiteral(track(" ", "}...`"), pos), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); // Adjusts 'pos' by accounting for the length of each portion of the string, // but only return the last given string @@ -364,22 +366,22 @@ describe('Colorization', function () { it("classifies partially written generics correctly.", function () { testLexicalClassification("Foo { @@ -400,7 +402,7 @@ describe('Colorization', function () { v = 2;\r\n\ >>>>>>> Branch - a\r\n\ }", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, keyword("class"), identifier("C"), punctuation("{"), @@ -412,7 +414,7 @@ describe('Colorization', function () { comment("=======\r\n v = 2;\r\n"), comment(">>>>>>> Branch - a"), punctuation("}"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); testLexicalClassification( "<<<<<<< HEAD\r\n\ @@ -420,7 +422,7 @@ class C { }\r\n\ =======\r\n\ class D { }\r\n\ >>>>>>> Branch - a\r\n", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, comment("<<<<<<< HEAD"), keyword("class"), identifier("C"), @@ -428,12 +430,12 @@ class D { }\r\n\ punctuation("}"), comment("=======\r\nclass D { }\r\n"), comment(">>>>>>> Branch - a"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); it("'of' keyword", function () { testLexicalClassification("for (var of of of) { }", - ts.EndOfLineState.Start, + ts.EndOfLineState.None, keyword("for"), punctuation("("), keyword("var"), @@ -443,7 +445,7 @@ class D { }\r\n\ punctuation(")"), punctuation("{"), punctuation("}"), - finalEndOfLineState(ts.EndOfLineState.Start)); + finalEndOfLineState(ts.EndOfLineState.None)); }); }); }); \ No newline at end of file