From ad9a87dfb6e9d9c7b811e92c2320ae306bbdc115 Mon Sep 17 00:00:00 2001 From: Mohamed Hegazy Date: Fri, 8 Aug 2014 10:37:10 -0700 Subject: [PATCH] Add some huristic optimization to not colorize a keyword if precceded by a dot or a keyword. this should handel cases for "a.var" or "module string { }" --- src/services/services.ts | 83 +++++++++++-------- .../cases/unittests/services/colorization.ts | 33 +++++++- 2 files changed, 82 insertions(+), 34 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index f00825e4b41..63feb8fed42 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -659,6 +659,8 @@ module ts { InMultiLineCommentTrivia, InSingleQuoteStringLiteral, InDoubleQuoteStringLiteral, + EndingWithKeyword, + EndingWithDotToken, } export enum TokenClass { @@ -2217,25 +2219,33 @@ module ts { function getClassificationsForLine(text: string, lexState: EndOfLineState): ClassificationResult { var offset = 0; var lastTokenOrCommentEnd = 0; - var inMultiLineComment = false; + var lastToken = SyntaxKind.Unknown; + var inUnterminatedMultiLineComment = false; - if (lexState !== EndOfLineState.Start) { - // If we're in a string literal, then prepend: "\ - // (and a newline). That way when we lex we'll think we're still in a string literal. - // - // If we're in a multiline comment, then prepend: /* - // (and a newline). That way when we lex we'll think we're still in a multiline comment. - if (lexState === EndOfLineState.InDoubleQuoteStringLiteral) { + // If we're in a string literal, then prepend: "\ + // (and a newline). That way when we lex we'll think we're still in a string literal. + // + // If we're in a multiline comment, then prepend: /* + // (and a newline). That way when we lex we'll think we're still in a multiline comment. + switch (lexState) { + case EndOfLineState.InDoubleQuoteStringLiteral: text = '"\\\n' + text; - } - else if (lexState === EndOfLineState.InSingleQuoteStringLiteral) { + offset = 3; + break; + case EndOfLineState.InSingleQuoteStringLiteral: text = "'\\\n" + text; - } - else if (lexState === EndOfLineState.InMultiLineCommentTrivia) { + offset = 3; + break; + case EndOfLineState.InMultiLineCommentTrivia: text = "/*\n" + text; - } - - offset = 3; + offset = 3; + break; + case EndOfLineState.EndingWithDotToken: + lastToken = SyntaxKind.DotToken; + break; + case EndOfLineState.EndingWithKeyword: + lastToken = SyntaxKind.FirstKeyword; + break; } var result: ClassificationResult = { @@ -2245,11 +2255,8 @@ module ts { scanner = createScanner(ScriptTarget.ES5, text, onError, processComment); - var lastToken = SyntaxKind.Unknown; var token = SyntaxKind.Unknown; do { - inMultiLineComment = false; - token = scanner.scan(); if ((token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) && !noRegexTable[lastToken]) { @@ -2257,6 +2264,9 @@ module ts { token = SyntaxKind.RegularExpressionLiteral; } } + else if (isKeyword(token) && (isKeyword(lastToken) || lastToken === SyntaxKind.DotToken)) { + token = SyntaxKind.Identifier; + } lastToken = token; @@ -2268,7 +2278,7 @@ module ts { function onError(message: DiagnosticMessage): void { - inMultiLineComment = message.key === Diagnostics.Asterisk_Slash_expected.key; + inUnterminatedMultiLineComment = message.key === Diagnostics.Asterisk_Slash_expected.key; } function processComment(start: number, end: number) { @@ -2291,21 +2301,24 @@ module ts { if (end >= text.length) { // We're at the end. - if (inMultiLineComment) { + if (inUnterminatedMultiLineComment) { result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; - return; } - - if (token === SyntaxKind.StringLiteral) { + else if (token === SyntaxKind.StringLiteral) { var tokenText = scanner.getTokenText(); if (tokenText.length > 0 && tokenText.charCodeAt(tokenText.length - 1) === CharacterCodes.backslash) { var quoteChar = tokenText.charCodeAt(0); result.finalLexState = quoteChar === CharacterCodes.doubleQuote ? EndOfLineState.InDoubleQuoteStringLiteral : EndOfLineState.InSingleQuoteStringLiteral; - return; } } + else if (token === SyntaxKind.DotToken) { + result.finalLexState = EndOfLineState.EndingWithDotToken; + } + else if (isKeyword(token)) { + result.finalLexState = EndOfLineState.EndingWithKeyword; + } } } @@ -2331,8 +2344,8 @@ module ts { } } - function isBinaryExpressionOperatorToken(tokenKind: SyntaxKind): boolean { - switch (tokenKind) { + function isBinaryExpressionOperatorToken(token: SyntaxKind): boolean { + switch (token) { case SyntaxKind.AsteriskToken: case SyntaxKind.SlashToken: case SyntaxKind.PercentToken: @@ -2374,8 +2387,8 @@ module ts { } } - function isPrefixUnaryExpressionOperatorToken(tokenKind: SyntaxKind): boolean { - switch (tokenKind) { + function isPrefixUnaryExpressionOperatorToken(token: SyntaxKind): boolean { + switch (token) { case SyntaxKind.PlusToken: case SyntaxKind.MinusToken: case SyntaxKind.TildeToken: @@ -2388,18 +2401,22 @@ module ts { } } - function classFromKind(kind: SyntaxKind) { - if (kind >= SyntaxKind.FirstKeyword && kind <= SyntaxKind.LastKeyword) { + function isKeyword(token: SyntaxKind): boolean { + return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword; + } + + function classFromKind(token: SyntaxKind) { + if (isKeyword(token)) { return TokenClass.Keyword; } - else if (isBinaryExpressionOperatorToken(kind) || isPrefixUnaryExpressionOperatorToken(kind)) { + else if (isBinaryExpressionOperatorToken(token) || isPrefixUnaryExpressionOperatorToken(token)) { return TokenClass.Operator; } - else if (kind >= SyntaxKind.FirstPunctuation && kind <= SyntaxKind.LastPunctuation) { + else if (token >= SyntaxKind.FirstPunctuation && token <= SyntaxKind.LastPunctuation) { return TokenClass.Punctuation; } - switch (kind) { + switch (token) { case SyntaxKind.NumericLiteral: return TokenClass.NumberLiteral; case SyntaxKind.StringLiteral: diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index cba14ae6957..2b9afdb6ef1 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -28,15 +28,21 @@ describe('Colorization', function () { var classResult = myclassifier.getClassificationsForLine(code, initialEndOfLineState).split('\n'); var tuples: Classification[] = []; var i = 0; + var computedLength = 0; for (; i < classResult.length - 1; i += 2) { - tuples[i / 2] = { + var t = tuples[i / 2] = { length: parseInt(classResult[i]), class: parseInt(classResult[i + 1]) }; + + assert.isTrue(t.length > 0, "Result length should be greater than 0, got :" + t.length); + computedLength += t.length; } var finalEndOfLineState = classResult[classResult.length - 1]; + assert.equal(computedLength, code.length, "Expected accumilative length of all entries to match the length of the source. expected: " + code.length + ", but got: " + computedLength); + return { tuples: tuples, finalEndOfLineState: parseInt(finalEndOfLineState) @@ -209,4 +215,29 @@ describe('Colorization', function () { assert.equal(results.finalEndOfLineState, ts.EndOfLineState.InMultiLineCommentTrivia); }); }); + + describe("test cases for colorizing keywords", function () { + it("classifies keyword after a dot", function () { + var results = getClassifications("a.var", ts.EndOfLineState.Start); + verifyClassification(results.tuples[2], 3, ts.TokenClass.Identifier); + }); + + it("classifies keyword after a keyword", function () { + var results = getClassifications("module string", ts.EndOfLineState.Start); + verifyClassification(results.tuples[2], 6, ts.TokenClass.Identifier); + }); + + it("reports correct state with a line ending in a keyword", function () { + var results = getClassifications("module", ts.EndOfLineState.Start); + assert.equal(results.finalEndOfLineState, ts.EndOfLineState.EndingWithKeyword); + }); + + it("classifies keyword after a dot on previous line", function () { + var results = getClassifications("var", ts.EndOfLineState.EndingWithDotToken); + + assert.equal(results.tuples.length, 1); + verifyClassification(results.tuples[0], 3, ts.TokenClass.Identifier); + assert.equal(results.finalEndOfLineState, ts.EndOfLineState.Start); + }); + }); }); \ No newline at end of file