From 5fd0701ce52cd6ae99d5f48e78762df4cc039710 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 25 Nov 2014 14:37:40 -0800 Subject: [PATCH 01/15] Fixed bug where tagged templates with a literal adjacent to EOF showed sig help. --- src/compiler/emitter.ts | 4 ++-- src/compiler/parser.ts | 7 ++++--- src/services/signatureHelp.ts | 2 +- .../signatureHelpTaggedTemplatesNegatives2.ts | 11 +++++++++++ .../signatureHelpTaggedTemplatesNegatives3.ts | 11 +++++++++++ .../signatureHelpTaggedTemplatesNegatives4.ts | 11 +++++++++++ .../signatureHelpTaggedTemplatesNegatives5.ts | 11 +++++++++++ 7 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives2.ts create mode 100644 tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives3.ts create mode 100644 tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives4.ts create mode 100644 tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts diff --git a/src/compiler/emitter.ts b/src/compiler/emitter.ts index 5cdf98aa63e..4595c8cb475 100644 --- a/src/compiler/emitter.ts +++ b/src/compiler/emitter.ts @@ -715,7 +715,7 @@ module ts { }; } } - + function emitEnumDeclaration(node: EnumDeclaration) { if (resolver.isDeclarationVisible(node)) { emitJsDocComments(node); @@ -3558,7 +3558,7 @@ module ts { return leadingComments; } - + function getLeadingCommentsToEmit(node: Node) { // Emit the leading comments only if the parent's pos doesn't match because parent should take care of emitting these comments if (node.parent.kind === SyntaxKind.SourceFile || node.pos !== node.parent.pos) { diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index 8d7bf9e4fe7..f60fd051cb1 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -823,9 +823,10 @@ module ts { return false; } - // If we didn't end in a backtick, we must still be in the middle of a template. - // If we did, make sure that it's not the *initial* backtick. - return sourceText.charCodeAt(node.end - 1) !== CharacterCodes.backtick || node.text.length === 0; + // If we didn't end in a backtick, we must still be in the middle of a template literal, + // but if it's the *initial* backtick (whereby the token is 1 char long), then it's unclosed. + var width = node.end - getTokenPosOfNode(node); + return width < 2 || sourceText.charCodeAt(node.end - 1) !== CharacterCodes.backtick; } export function isModifier(token: SyntaxKind): boolean { diff --git a/src/services/signatureHelp.ts b/src/services/signatureHelp.ts index e2b65e3c6e1..aa2f5d32c50 100644 --- a/src/services/signatureHelp.ts +++ b/src/services/signatureHelp.ts @@ -296,7 +296,7 @@ module ts.SignatureHelp { Debug.assert(templateExpression.kind === SyntaxKind.TemplateExpression); // If we're just after a template tail, don't show signature help. - if (node.kind === SyntaxKind.TemplateTail && position >= node.getEnd() && !isUnterminatedTemplateEnd(node)) { + if (node.kind === SyntaxKind.TemplateTail && !isInsideTemplateLiteral(node, position)) { return undefined; } diff --git a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives2.ts b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives2.ts new file mode 100644 index 00000000000..e7a5078dcd0 --- /dev/null +++ b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives2.ts @@ -0,0 +1,11 @@ +/// + +//// function foo(strs, ...rest) { +//// } +//// +//// /*1*/fo/*2*/o /*3*/`abcd${0 + 1}abcd{1 + 1}`/*4*/ /*5*/ + +test.markers().forEach(m => { + goTo.position(m.position); + verify.not.signatureHelpPresent(); +}); \ No newline at end of file diff --git a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives3.ts b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives3.ts new file mode 100644 index 00000000000..c84b92fee4f --- /dev/null +++ b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives3.ts @@ -0,0 +1,11 @@ +/// + +//// function foo(strs, ...rest) { +//// } +//// +//// /*1*/fo/*2*/o /*3*/`abcd${0 + 1}abcd{1 + 1}abcd`/*4*/ /*5*/ + +test.markers().forEach(m => { + goTo.position(m.position); + verify.not.signatureHelpPresent(); +}); \ No newline at end of file diff --git a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives4.ts b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives4.ts new file mode 100644 index 00000000000..338a8dc65ca --- /dev/null +++ b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives4.ts @@ -0,0 +1,11 @@ +/// + +//// function foo(strs, ...rest) { +//// } +//// +//// /*1*/fo/*2*/o /*3*/``/*4*/ /*5*/ + +test.markers().forEach(m => { + goTo.position(m.position); + verify.not.signatureHelpPresent(); +}); \ No newline at end of file diff --git a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts new file mode 100644 index 00000000000..204a30fa1bd --- /dev/null +++ b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts @@ -0,0 +1,11 @@ +/// + +//// function foo(strs, ...rest) { +//// } +//// +//// /*1*/fo/*2*/o /*3*/`abcd`/*4*/ /*5*/ + +test.markers().forEach(m => { + goTo.position(m.position); + verify.not.signatureHelpPresent(); +}); \ No newline at end of file From 46991b79e04deeb3b3e49f7b0063275d271b4077 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 25 Nov 2014 14:42:15 -0800 Subject: [PATCH 02/15] Fixed test. --- tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts index 204a30fa1bd..02383df9267 100644 --- a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts +++ b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts @@ -3,7 +3,7 @@ //// function foo(strs, ...rest) { //// } //// -//// /*1*/fo/*2*/o /*3*/`abcd`/*4*/ /*5*/ +//// /*1*/fo/*2*/o /*3*/`abcd`/*4*/ test.markers().forEach(m => { goTo.position(m.position); From ea30c68128499f649be64711ae94b7946d16acff Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 19 Jan 2015 18:07:54 -0800 Subject: [PATCH 03/15] Rudimentary, but imperfect, lexical classification for templates. --- src/services/services.ts | 95 +++++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index def895880e6..d585722e7ef 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1143,6 +1143,9 @@ module ts { InMultiLineCommentTrivia, InSingleQuoteStringLiteral, InDoubleQuoteStringLiteral, + InTemplateHeadLiteral, // this could also be a NoSubstitutionTemplateLiteral + InTemplateMiddleLiteral, //this could also be a TemplateTail + InTemplateSubstitutionPosition, } export enum TokenClass { @@ -5650,12 +5653,12 @@ module ts { // if there are more cases we want the classifier to be better at. return true; } - - // 'classifyKeywordsInGenerics' should be 'true' when a syntactic classifier is not present. - function getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult { + + function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult { var offset = 0; var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; + var templateStack: SyntaxKind[]; // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. @@ -5675,6 +5678,21 @@ module ts { text = "/*\n" + text; offset = 3; break; + case EndOfLineState.InTemplateHeadLiteral: + if (syntacticClassifierAbsent) { + text = "`\n" + text; + offset = 2; + } + break; + case EndOfLineState.InTemplateMiddleLiteral: + if (syntacticClassifierAbsent) { + text = "${\n" + text; + offset = 3; + } + // fallthrough + case EndOfLineState.InTemplateSubstitutionPosition: + templateStack = [SyntaxKind.TemplateHead]; + break; } scanner.setText(text); @@ -5739,12 +5757,50 @@ module ts { token === SyntaxKind.StringKeyword || token === SyntaxKind.NumberKeyword || token === SyntaxKind.BooleanKeyword) { - if (angleBracketStack > 0 && !classifyKeywordsInGenerics) { - // If it looks like we're could be in something generic, don't classify this - // as a keyword. We may just get overwritten by the syntactic classifier, - // causing a noisy experience for the user. - token = SyntaxKind.Identifier; - } + if (angleBracketStack > 0 && !syntacticClassifierAbsent) { + // If it looks like we're could be in something generic, don't classify this + // as a keyword. We may just get overwritten by the syntactic classifier, + // causing a noisy experience for the user. + token = SyntaxKind.Identifier; + } + } + else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { + if (!templateStack) { + templateStack = [token]; + } + else { + templateStack.push(token); + } + } + else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack && templateStack.length > 0) { + templateStack.push(token); + } + } + else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack && templateStack.length > 0) { + var lastTemplateStackToken = lastOrUndefined(templateStack); + + if (lastTemplateStackToken === SyntaxKind.TemplateHead) { + token = scanner.reScanTemplateToken(); + + // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us. + if (token === SyntaxKind.TemplateTail) { + templateStack.pop(); + } + else { + Debug.assert(token === SyntaxKind.TemplateMiddle, "Should have been a template middle. Was " + token); + } + } + else { + Debug.assert(token === SyntaxKind.CloseBraceToken, "Should have been an open brace. Was: " + token); + templateStack.pop(); + } + } } lastNonTriviaToken = token; @@ -5760,7 +5816,7 @@ module ts { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); - addResult(end - start, classFromKind(token)); + addResult(end - start, classFromKind(token, syntacticClassifierAbsent)); if (end >= text.length) { if (token === SyntaxKind.StringLiteral) { @@ -5789,6 +5845,19 @@ module ts { result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; } } + else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { + if (scanner.isUnterminated()) { + if (token === SyntaxKind.TemplateMiddle) { + result.finalLexState = EndOfLineState.InTemplateMiddleLiteral; + } + else { + result.finalLexState = EndOfLineState.InTemplateHeadLiteral; + } + } + } + else if (templateStack && templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { + result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition; + } } } @@ -5866,7 +5935,7 @@ module ts { return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword; } - function classFromKind(token: SyntaxKind) { + function classFromKind(token: SyntaxKind, syntacticClassifierAbsent?: boolean) { if (isKeyword(token)) { return TokenClass.Keyword; } @@ -5892,6 +5961,10 @@ module ts { return TokenClass.Whitespace; case SyntaxKind.Identifier: default: + // Only give a classification if nothing will more accurately classify. + if (syntacticClassifierAbsent && isTemplateLiteralKind(token)) { + return TokenClass.StringLiteral; // should make a TemplateLiteral + } return TokenClass.Identifier; } } From 3429fab6fbdd5fb37975c088b88079a4cd22fb30 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 20 Jan 2015 12:42:03 -0800 Subject: [PATCH 04/15] Fixed issue with the kinds we check. --- src/services/services.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index d585722e7ef..50b852c241b 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1143,8 +1143,8 @@ module ts { InMultiLineCommentTrivia, InSingleQuoteStringLiteral, InDoubleQuoteStringLiteral, - InTemplateHeadLiteral, // this could also be a NoSubstitutionTemplateLiteral - InTemplateMiddleLiteral, //this could also be a TemplateTail + InTemplateHeadOrNoSubstitutionTemplate, + InTemplateMiddleOrTail, InTemplateSubstitutionPosition, } @@ -5678,20 +5678,22 @@ module ts { text = "/*\n" + text; offset = 3; break; - case EndOfLineState.InTemplateHeadLiteral: + case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate: if (syntacticClassifierAbsent) { text = "`\n" + text; offset = 2; } break; - case EndOfLineState.InTemplateMiddleLiteral: + case EndOfLineState.InTemplateMiddleOrTail: if (syntacticClassifierAbsent) { - text = "${\n" + text; - offset = 3; + text = "}\n" + text; + offset = 2; } // fallthrough case EndOfLineState.InTemplateSubstitutionPosition: - templateStack = [SyntaxKind.TemplateHead]; + if (syntacticClassifierAbsent) { + templateStack = [SyntaxKind.TemplateHead]; + } break; } @@ -5847,11 +5849,14 @@ module ts { } else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { if (scanner.isUnterminated()) { - if (token === SyntaxKind.TemplateMiddle) { - result.finalLexState = EndOfLineState.InTemplateMiddleLiteral; + if (token === SyntaxKind.TemplateTail) { + result.finalLexState = EndOfLineState.InTemplateMiddleOrTail; + } + else if (token === SyntaxKind.NoSubstitutionTemplateLiteral) { + result.finalLexState = EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate; } else { - result.finalLexState = EndOfLineState.InTemplateHeadLiteral; + Debug.fail("Only 'NoSubstitutionTemplateLiteral's and 'TemplateTail's can be unterminated; got SyntaxKind #" + token); } } } From f1f085eda65876cb3d8ae2a57b3ec8a03f86ac7b Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 12:06:01 -0800 Subject: [PATCH 05/15] Addressed CR feedback. --- src/services/services.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 50b852c241b..858a1ac1757 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5620,6 +5620,10 @@ module ts { noRegexTable[SyntaxKind.TrueKeyword] = true; noRegexTable[SyntaxKind.FalseKeyword] = true; + // Just a stack of TemplateHeads and OpenCurlyBraces, used + // to perform rudimentary classification on templates. + var templateStack: SyntaxKind[] = []; + function isAccessibilityModifier(kind: SyntaxKind) { switch (kind) { case SyntaxKind.PublicKeyword: @@ -5658,7 +5662,6 @@ module ts { var offset = 0; var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; - var templateStack: SyntaxKind[]; // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. @@ -5725,6 +5728,11 @@ module ts { // work well enough in practice. var angleBracketStack = 0; + // Empty out the template stack for reuse. + while (templateStack.length > 0) { + templateStack.pop(); + } + do { token = scanner.scan(); @@ -5767,24 +5775,19 @@ module ts { } } else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { - if (!templateStack) { - templateStack = [token]; - } - else { - templateStack.push(token); - } + templateStack.push(token); } else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. - if (templateStack && templateStack.length > 0) { + if (templateStack.length > 0) { templateStack.push(token); } } else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. - if (templateStack && templateStack.length > 0) { + if (templateStack.length > 0) { var lastTemplateStackToken = lastOrUndefined(templateStack); if (lastTemplateStackToken === SyntaxKind.TemplateHead) { @@ -5860,7 +5863,7 @@ module ts { } } } - else if (templateStack && templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { + else if (templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition; } } From 3fea0aefbef3ff808de8799bd1a622cd41c08885 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 12:37:03 -0800 Subject: [PATCH 06/15] classifyKeywordsInGenerics -> syntacticClassifierAbsent --- src/services/services.ts | 2 +- src/services/shims.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 858a1ac1757..172fb4ab00e 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1171,7 +1171,7 @@ module ts { } export interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** diff --git a/src/services/shims.ts b/src/services/shims.ts index 0f9f1a12cd1..a0eb1cb1ac4 100644 --- a/src/services/shims.ts +++ b/src/services/shims.ts @@ -165,7 +165,7 @@ module ts { } export interface ClassifierShim extends Shim { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): string; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string; } export interface CoreServicesShim extends Shim { From ab79faef85568a9944709c0940c6f484aaa5187c Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 15:41:01 -0800 Subject: [PATCH 07/15] Added tests, fixed order of emptying templateStack, unconditionally perform template classification. --- src/services/services.ts | 43 +++--- .../cases/unittests/services/colorization.ts | 130 ++++++++++++++++-- 2 files changed, 137 insertions(+), 36 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 172fb4ab00e..673748d6969 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5663,6 +5663,11 @@ module ts { var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; + // Empty out the template stack for reuse. + while (templateStack.length > 0) { + templateStack.pop(); + } + // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. // @@ -5682,21 +5687,15 @@ module ts { offset = 3; break; case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate: - if (syntacticClassifierAbsent) { - text = "`\n" + text; - offset = 2; - } + text = "`\n" + text; + offset = 2; break; case EndOfLineState.InTemplateMiddleOrTail: - if (syntacticClassifierAbsent) { - text = "}\n" + text; - offset = 2; - } + text = "}\n" + text; + offset = 2; // fallthrough case EndOfLineState.InTemplateSubstitutionPosition: - if (syntacticClassifierAbsent) { - templateStack = [SyntaxKind.TemplateHead]; - } + templateStack.push(SyntaxKind.TemplateHead); break; } @@ -5728,11 +5727,6 @@ module ts { // work well enough in practice. var angleBracketStack = 0; - // Empty out the template stack for reuse. - while (templateStack.length > 0) { - templateStack.pop(); - } - do { token = scanner.scan(); @@ -5774,17 +5768,17 @@ module ts { token = SyntaxKind.Identifier; } } - else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { + else if (token === SyntaxKind.TemplateHead) { templateStack.push(token); } - else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { + else if (token === SyntaxKind.OpenBraceToken) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. if (templateStack.length > 0) { templateStack.push(token); } } - else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { + else if (token === SyntaxKind.CloseBraceToken) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. if (templateStack.length > 0) { @@ -5821,7 +5815,7 @@ module ts { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); - addResult(end - start, classFromKind(token, syntacticClassifierAbsent)); + addResult(end - start, classFromKind(token)); if (end >= text.length) { if (token === SyntaxKind.StringLiteral) { @@ -5850,7 +5844,7 @@ module ts { result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; } } - else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { + else if (isTemplateLiteralKind(token)) { if (scanner.isUnterminated()) { if (token === SyntaxKind.TemplateTail) { result.finalLexState = EndOfLineState.InTemplateMiddleOrTail; @@ -5943,7 +5937,7 @@ module ts { return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword; } - function classFromKind(token: SyntaxKind, syntacticClassifierAbsent?: boolean) { + function classFromKind(token: SyntaxKind) { if (isKeyword(token)) { return TokenClass.Keyword; } @@ -5969,9 +5963,8 @@ module ts { return TokenClass.Whitespace; case SyntaxKind.Identifier: default: - // Only give a classification if nothing will more accurately classify. - if (syntacticClassifierAbsent && isTemplateLiteralKind(token)) { - return TokenClass.StringLiteral; // should make a TemplateLiteral + if (isTemplateLiteralKind(token)) { + return TokenClass.StringLiteral; // maybe make a TemplateLiteral } return TokenClass.Identifier; } diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index 01149764305..c834f85806d 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -4,6 +4,7 @@ interface ClassificationEntry { value: any; classification: ts.TokenClass; + position?: number; } describe('Colorization', function () { @@ -23,16 +24,23 @@ describe('Colorization', function () { return undefined; } - function punctuation(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Punctuation }; } - function keyword(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Keyword }; } - function operator(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Operator }; } - function comment(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Comment }; } - function whitespace(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Whitespace }; } - function identifier(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Identifier }; } - function numberLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.NumberLiteral }; } - function stringLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.StringLiteral }; } - function regExpLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.RegExpLiteral }; } - function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined }; } + function punctuation(text: string, position?: number) { return createClassification(text, ts.TokenClass.Punctuation, position); } + function keyword(text: string, position?: number) { return createClassification(text, ts.TokenClass.Keyword, position); } + function operator(text: string, position?: number) { return createClassification(text, ts.TokenClass.Operator, position); } + function comment(text: string, position?: number) { return createClassification(text, ts.TokenClass.Comment, position); } + function whitespace(text: string, position?: number) { return createClassification(text, ts.TokenClass.Whitespace, position); } + function identifier(text: string, position?: number) { return createClassification(text, ts.TokenClass.Identifier, position); } + function numberLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.NumberLiteral, position); } + function stringLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.StringLiteral, position); } + function regExpLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.RegExpLiteral, position); } + function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined, position: 0 }; } + function createClassification(text: string, tokenClass: ts.TokenClass, position?: number): ClassificationEntry { + return { + value: text, + classification: tokenClass, + position: position, + }; + } function testLexicalClassification(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void { var result = classifier.getClassificationsForLine(text, initialEndOfLineState); @@ -44,7 +52,7 @@ describe('Colorization', function () { assert.equal(result.finalLexState, expectedEntry.value, "final endOfLineState does not match expected."); } else { - var actualEntryPosition = text.indexOf(expectedEntry.value); + var actualEntryPosition = expectedEntry.position !== undefined ? expectedEntry.position : text.indexOf(expectedEntry.value); assert(actualEntryPosition >= 0, "token: '" + expectedEntry.value + "' does not exit in text: '" + text + "'."); var actualEntry = getEntryAtPosistion(result, actualEntryPosition); @@ -254,6 +262,106 @@ describe('Colorization', function () { finalEndOfLineState(ts.EndOfLineState.Start)); }); + it("classifies a single line no substitution template string correctly", () => { + testLexicalClassification("`number number public string`", + ts.EndOfLineState.Start, + stringLiteral("`number number public string`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution parts of a template string correctly", () => { + testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`", + ts.EndOfLineState.Start, + stringLiteral("`number '${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}' string '${"), + stringLiteral("'hello'"), + stringLiteral("}'`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies an unterminated no substitution template string correctly", () => { + testLexicalClassification("`hello world", + ts.EndOfLineState.Start, + stringLiteral("`hello world"), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline no-substitution/head template", () => { + testLexicalClassification("...", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline template middle/end",() => { + testLexicalClassification("...", + ts.EndOfLineState.InTemplateMiddleOrTail, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateMiddleOrTail)); + }); + it("classifies a termination of a multiline template head", () => { + testLexicalClassification("...${", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...${"), + finalEndOfLineState(ts.EndOfLineState.InTemplateSubstitutionPosition)); + }); + it("classifies the termination of a multiline no substitution template", () => { + testLexicalClassification("...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies the substitution parts and middle/tail of a multiline template string", () => { + testLexicalClassification("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies a template middle and propagates the end of line state",() => { + testLexicalClassification("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution expressions with curly braces appropriately", () => { + var pos = 0; + var lastLength = 0; + + testLexicalClassification("...${ () => { } } ${ { x: `1` } }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral(track("...${"), pos), + punctuation(track(" ", "("), pos), + punctuation(track(")"), pos), + punctuation(track(" ", "=>"), pos), + punctuation(track(" ", "{"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "} ${"), pos), + punctuation(track(" ", "{"), pos), + identifier(track(" ", "x"), pos), + punctuation(track(":"), pos), + stringLiteral(track(" ", "`1`"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "}...`"), pos), + finalEndOfLineState(ts.EndOfLineState.Start)); + + // Adjusts 'pos' by accounting for the length of each portion of the string, + // but only return the last given string + function track(...vals: string[]): string { + for (var i = 0, n = vals.length; i < n; i++) { + pos += lastLength; + lastLength = vals[i].length; + } + return ts.lastOrUndefined(vals); + } + }); + it("classifies partially written generics correctly.", function () { testLexicalClassification("Foo Date: Mon, 26 Jan 2015 15:43:26 -0800 Subject: [PATCH 08/15] Fixed assertion --- src/services/services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/services.ts b/src/services/services.ts index 673748d6969..90227eeb1f7 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5796,7 +5796,7 @@ module ts { } } else { - Debug.assert(token === SyntaxKind.CloseBraceToken, "Should have been an open brace. Was: " + token); + Debug.assert(lastTemplateStackToken === SyntaxKind.OpenBraceToken, "Should have been an open brace. Was: " + token); templateStack.pop(); } } From 15b333e10281fcb3d136b57d80597233154d1742 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 16:06:56 -0800 Subject: [PATCH 09/15] Added comment for 'syntacticClassifierAbsent'. --- src/services/services.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/services/services.ts b/src/services/services.ts index 90227eeb1f7..39eab2f7e16 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5658,6 +5658,8 @@ module ts { return true; } + // If there is a syntactic classifier ('syntacticClassifierAbsent' is false), + // we will be more conservative in order to avoid conflicting with the syntactic classifier. function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult { var offset = 0; var token = SyntaxKind.Unknown; From 65e23b1e60699ed22add237a75ada0e3589f4aa2 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Thu, 12 Feb 2015 16:46:25 -0800 Subject: [PATCH 10/15] Added API baseline changes. --- tests/baselines/reference/APISample_compile.js | 5 ++++- tests/baselines/reference/APISample_compile.types | 15 ++++++++++++--- tests/baselines/reference/APISample_linter.js | 5 ++++- tests/baselines/reference/APISample_linter.types | 15 ++++++++++++--- tests/baselines/reference/APISample_transform.js | 5 ++++- .../baselines/reference/APISample_transform.types | 15 ++++++++++++--- tests/baselines/reference/APISample_watcher.js | 5 ++++- tests/baselines/reference/APISample_watcher.types | 15 ++++++++++++--- 8 files changed, 64 insertions(+), 16 deletions(-) diff --git a/tests/baselines/reference/APISample_compile.js b/tests/baselines/reference/APISample_compile.js index 9a44ae12df0..98b80ebc868 100644 --- a/tests/baselines/reference/APISample_compile.js +++ b/tests/baselines/reference/APISample_compile.js @@ -1741,6 +1741,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1762,7 +1765,7 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_compile.types b/tests/baselines/reference/APISample_compile.types index 69458320236..77958b6b16a 100644 --- a/tests/baselines/reference/APISample_compile.types +++ b/tests/baselines/reference/APISample_compile.types @@ -5539,6 +5539,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5594,12 +5603,12 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/baselines/reference/APISample_linter.js b/tests/baselines/reference/APISample_linter.js index 5413cdb8a78..f22532d353d 100644 --- a/tests/baselines/reference/APISample_linter.js +++ b/tests/baselines/reference/APISample_linter.js @@ -1772,6 +1772,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1793,7 +1796,7 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_linter.types b/tests/baselines/reference/APISample_linter.types index 55613f8d3e3..09b242ac043 100644 --- a/tests/baselines/reference/APISample_linter.types +++ b/tests/baselines/reference/APISample_linter.types @@ -5683,6 +5683,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5738,12 +5747,12 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/baselines/reference/APISample_transform.js b/tests/baselines/reference/APISample_transform.js index 13ec31cd7d2..56d7e7db5bc 100644 --- a/tests/baselines/reference/APISample_transform.js +++ b/tests/baselines/reference/APISample_transform.js @@ -1773,6 +1773,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1794,7 +1797,7 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_transform.types b/tests/baselines/reference/APISample_transform.types index a8ba6e9f521..ad9623c570e 100644 --- a/tests/baselines/reference/APISample_transform.types +++ b/tests/baselines/reference/APISample_transform.types @@ -5635,6 +5635,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5690,12 +5699,12 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/baselines/reference/APISample_watcher.js b/tests/baselines/reference/APISample_watcher.js index 5c3f944faf1..f03a36cfc2e 100644 --- a/tests/baselines/reference/APISample_watcher.js +++ b/tests/baselines/reference/APISample_watcher.js @@ -1810,6 +1810,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1831,7 +1834,7 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_watcher.types b/tests/baselines/reference/APISample_watcher.types index 54304c2e1b9..41f6fd1ced6 100644 --- a/tests/baselines/reference/APISample_watcher.types +++ b/tests/baselines/reference/APISample_watcher.types @@ -5808,6 +5808,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5863,12 +5872,12 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** From be40cac7fdb418d772dd636560ecb526dd98c253 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 12 Feb 2015 17:25:57 -0800 Subject: [PATCH 11/15] Don't check invariants when running RWC tests. It adds too much running time. --- src/harness/harness.ts | 15 +++++++++------ src/harness/rwcRunner.ts | 3 ++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/harness/harness.ts b/src/harness/harness.ts index 534bf85d119..86a68af1145 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -795,9 +795,12 @@ module Harness { } } - export function createSourceFileAndAssertInvariants(fileName: string, sourceText: string, languageVersion: ts.ScriptTarget) { - var result = ts.createSourceFile(fileName, sourceText, languageVersion, /*setParentNodes:*/ true); - Utils.assertInvariants(result, /*parent:*/ undefined); + export function createSourceFileAndAssertInvariants(fileName: string, sourceText: string, languageVersion: ts.ScriptTarget, assertInvariants = true) { + // Only set the parent nodes if we're asserting invariants. We don't need them otherwise. + var result = ts.createSourceFile(fileName, sourceText, languageVersion, /*setParentNodes:*/ assertInvariants); + if (assertInvariants) { + Utils.assertInvariants(result, /*parent:*/ undefined); + } return result; } @@ -805,7 +808,6 @@ module Harness { export var defaultLibSourceFile = createSourceFileAndAssertInvariants(defaultLibFileName, IO.readFile(libFolder + 'lib.core.d.ts'), /*languageVersion*/ ts.ScriptTarget.Latest); export var defaultES6LibSourceFile = createSourceFileAndAssertInvariants(defaultLibFileName, IO.readFile(libFolder + 'lib.core.es6.d.ts'), /*languageVersion*/ ts.ScriptTarget.Latest); - // Cache these between executions so we don't have to re-parse them for every test export var fourslashFileName = 'fourslash.ts'; export var fourslashSourceFile: ts.SourceFile; @@ -926,7 +928,8 @@ module Harness { settingsCallback?: (settings: ts.CompilerOptions) => void, options?: ts.CompilerOptions, // Current directory is needed for rwcRunner to be able to use currentDirectory defined in json file - currentDirectory?: string) { + currentDirectory?: string, + assertInvariants = true) { options = options || { noResolve: false }; options.target = options.target || ts.ScriptTarget.ES3; @@ -1074,7 +1077,7 @@ module Harness { var register = (file: { unitName: string; content: string; }) => { if (file.content !== undefined) { var fileName = ts.normalizeSlashes(file.unitName); - filemap[getCanonicalFileName(fileName)] = createSourceFileAndAssertInvariants(fileName, file.content, options.target); + filemap[getCanonicalFileName(fileName)] = createSourceFileAndAssertInvariants(fileName, file.content, options.target, assertInvariants); } }; inputFiles.forEach(register); diff --git a/src/harness/rwcRunner.ts b/src/harness/rwcRunner.ts index b66322f8390..c1c3251cf02 100644 --- a/src/harness/rwcRunner.ts +++ b/src/harness/rwcRunner.ts @@ -90,7 +90,8 @@ module RWC { /*settingsCallback*/ undefined, opts.options, // Since all Rwc json file specified current directory in its json file, we need to pass this information to compilerHost // so that when the host is asked for current directory, it should give the value from json rather than from process - currentDirectory); + currentDirectory, + /*assertInvariants:*/ false); }); function getHarnessCompilerInputUnit(fileName: string) { From b1dc910b4a349e402efe0a1721b73d0bfa9bb610 Mon Sep 17 00:00:00 2001 From: Bill Ticehurst Date: Thu, 12 Feb 2015 18:39:59 -0800 Subject: [PATCH 12/15] Added script to easily enable VS dev mode --- scripts/VSDevMode.ps1 | 49 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 scripts/VSDevMode.ps1 diff --git a/scripts/VSDevMode.ps1 b/scripts/VSDevMode.ps1 new file mode 100644 index 00000000000..21cc1f48ac1 --- /dev/null +++ b/scripts/VSDevMode.ps1 @@ -0,0 +1,49 @@ +<# +.SYNOPSIS +Run this PowerShell script to enable dev mode and/or a custom script for the TypeScript language service, e.g. + +PS C:\> .\scripts\VSDevMode.ps1 -enableDevMode -tsScript C:\src\TypeScript\built\local\typescriptServices.js + +Note: If you get security errors, try running powershell as an Administrator and with the "-executionPolicy remoteSigned" switch + +.PARAMETER vsVersion +Set to "12" for Dev12 (VS2013) or "14" (the default) for Dev14 (VS2015) + +.PARAMETER enableDevMode +Pass this switch to enable attaching a debugger to the language service + +.PARAMETER tsScript +The path to a custom language service script to use, e.g. "C:\src\TypeScript\built\local\typescriptServices.js" +#> +Param( + [int]$vsVersion = 14, + [switch]$enableDevMode, + [string]$tsScript +) + +$vsRegKey = "HKCU:\Software\Microsoft\VisualStudio\${vsVersion}.0" +$tsRegKey = "${vsRegKey}\TypeScriptLanguageService" + +if($enableDevMode -ne $true -and $tsScript -eq ""){ + Throw "You must either enable language service debugging (-enableDevMode), set a custom script (-tsScript), or both" +} + +if(!(Test-Path $vsRegKey)){ + Throw "Visual Studio ${vsVersion} is not installed" +} +if(!(Test-Path $tsRegKey)){ + # Create the TypeScript subkey if it doesn't exist + New-Item -path $tsRegKey +} + +if($tsScript -ne ""){ + if(!(Test-Path $tsScript)){ + Throw "Could not locate the TypeScript language service script at ${tsScript}" + } + Set-ItemProperty -path $tsRegKey -name CustomTypeScriptServicesFileLocation -value "${tsScript}" + Write-Host "Enabled custom TypeScript language service at ${tsScript} for Dev${vsVersion}" +} +if($enableDevMode){ + Set-ItemProperty -path $tsRegKey -name EnableDevMode -value 1 + Write-Host "Enabled developer mode for Dev${vsVersion}" +} From 1dca5620f52d13af066187cd39357cdfcc9a6123 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Fri, 13 Feb 2015 12:56:21 -0800 Subject: [PATCH 13/15] Addressed CR feedback. --- src/services/services.ts | 45 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 39eab2f7e16..0a901f0df0b 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1171,7 +1171,26 @@ module ts { } export interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** @@ -5620,8 +5639,26 @@ module ts { noRegexTable[SyntaxKind.TrueKeyword] = true; noRegexTable[SyntaxKind.FalseKeyword] = true; - // Just a stack of TemplateHeads and OpenCurlyBraces, used - // to perform rudimentary classification on templates. + // Just a stack of TemplateHeads and OpenCurlyBraces, used to perform rudimentary (inexact) + // classification on template strings. Because of the context free nature of templates, + // the only precise way to classify a template portion would be by propagating the stack across + // lines, just as we do with the end-of-line state. However, this is a burden for implementers, + // and the behavior is entirely subsumed by the syntactic classifier anyway, so we instead + // flatten any nesting when the template stack is non-empty and encode it in the end-of-line state. + // Situations in which this fails are + // 1) When template strings are nested across different lines: + // `hello ${ `world + // ` }` + // + // Where on the second line, you will get the closing of a template, + // a closing curly, and a new template. + // + // 2) When substitution expressions have curly braces and the curly brace falls on the next line: + // `hello ${ () => { + // return "world" } } ` + // + // Where on the second line, you will get the 'return' keyword, + // a string literal, and a template end consisting of '} } `'. var templateStack: SyntaxKind[] = []; function isAccessibilityModifier(kind: SyntaxKind) { @@ -5966,7 +6003,7 @@ module ts { case SyntaxKind.Identifier: default: if (isTemplateLiteralKind(token)) { - return TokenClass.StringLiteral; // maybe make a TemplateLiteral + return TokenClass.StringLiteral; } return TokenClass.Identifier; } From a1b90f0e323a93de4e28f2bdcab3b4e15cb0593d Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Fri, 13 Feb 2015 13:33:12 -0800 Subject: [PATCH 14/15] Added baselines. --- .../baselines/reference/APISample_compile.js | 21 ++++++++++++++++- .../reference/APISample_compile.types | 23 +++++++++++++++++-- tests/baselines/reference/APISample_linter.js | 21 ++++++++++++++++- .../reference/APISample_linter.types | 23 +++++++++++++++++-- .../reference/APISample_transform.js | 21 ++++++++++++++++- .../reference/APISample_transform.types | 23 +++++++++++++++++-- .../baselines/reference/APISample_watcher.js | 21 ++++++++++++++++- .../reference/APISample_watcher.types | 23 +++++++++++++++++-- 8 files changed, 164 insertions(+), 12 deletions(-) diff --git a/tests/baselines/reference/APISample_compile.js b/tests/baselines/reference/APISample_compile.js index 98b80ebc868..b91a706aa45 100644 --- a/tests/baselines/reference/APISample_compile.js +++ b/tests/baselines/reference/APISample_compile.js @@ -1765,7 +1765,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_compile.types b/tests/baselines/reference/APISample_compile.types index 77958b6b16a..ae53c666807 100644 --- a/tests/baselines/reference/APISample_compile.types +++ b/tests/baselines/reference/APISample_compile.types @@ -5603,8 +5603,27 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState diff --git a/tests/baselines/reference/APISample_linter.js b/tests/baselines/reference/APISample_linter.js index f22532d353d..68c13edb7db 100644 --- a/tests/baselines/reference/APISample_linter.js +++ b/tests/baselines/reference/APISample_linter.js @@ -1796,7 +1796,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_linter.types b/tests/baselines/reference/APISample_linter.types index 09b242ac043..0c153e7b1dc 100644 --- a/tests/baselines/reference/APISample_linter.types +++ b/tests/baselines/reference/APISample_linter.types @@ -5747,8 +5747,27 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState diff --git a/tests/baselines/reference/APISample_transform.js b/tests/baselines/reference/APISample_transform.js index 56d7e7db5bc..0e80f6b55b9 100644 --- a/tests/baselines/reference/APISample_transform.js +++ b/tests/baselines/reference/APISample_transform.js @@ -1797,7 +1797,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_transform.types b/tests/baselines/reference/APISample_transform.types index ad9623c570e..0c201b0a5fd 100644 --- a/tests/baselines/reference/APISample_transform.types +++ b/tests/baselines/reference/APISample_transform.types @@ -5699,8 +5699,27 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState diff --git a/tests/baselines/reference/APISample_watcher.js b/tests/baselines/reference/APISample_watcher.js index f03a36cfc2e..27a2ce5e415 100644 --- a/tests/baselines/reference/APISample_watcher.js +++ b/tests/baselines/reference/APISample_watcher.js @@ -1834,7 +1834,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_watcher.types b/tests/baselines/reference/APISample_watcher.types index 41f6fd1ced6..4e137cae59d 100644 --- a/tests/baselines/reference/APISample_watcher.types +++ b/tests/baselines/reference/APISample_watcher.types @@ -5872,8 +5872,27 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState From 91a39dd000844b4171fba073cc09adf4203fa0f2 Mon Sep 17 00:00:00 2001 From: Yui T Date: Fri, 13 Feb 2015 13:42:17 -0800 Subject: [PATCH 15/15] Fix time measurement --- src/compiler/tsc.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/tsc.ts b/src/compiler/tsc.ts index 4bcd0856ca8..3f7e6362e19 100644 --- a/src/compiler/tsc.ts +++ b/src/compiler/tsc.ts @@ -332,7 +332,7 @@ module ts { var program = createProgram(fileNames, compilerOptions, compilerHost); var exitStatus = compileProgram(); - var end = start - new Date().getTime(); + var end = new Date().getTime() - start; if (compilerOptions.listFiles) { forEach(program.getSourceFiles(), file => { @@ -357,7 +357,7 @@ module ts { reportTimeStatistic("Bind time", ts.bindTime); reportTimeStatistic("Check time", ts.checkTime); reportTimeStatistic("Emit time", ts.emitTime); - reportTimeStatistic("Total time", start - end); + reportTimeStatistic("Total time", end); } return { program, exitStatus };