diff --git a/scripts/VSDevMode.ps1 b/scripts/VSDevMode.ps1 new file mode 100644 index 00000000000..21cc1f48ac1 --- /dev/null +++ b/scripts/VSDevMode.ps1 @@ -0,0 +1,49 @@ +<# +.SYNOPSIS +Run this PowerShell script to enable dev mode and/or a custom script for the TypeScript language service, e.g. + +PS C:\> .\scripts\VSDevMode.ps1 -enableDevMode -tsScript C:\src\TypeScript\built\local\typescriptServices.js + +Note: If you get security errors, try running powershell as an Administrator and with the "-executionPolicy remoteSigned" switch + +.PARAMETER vsVersion +Set to "12" for Dev12 (VS2013) or "14" (the default) for Dev14 (VS2015) + +.PARAMETER enableDevMode +Pass this switch to enable attaching a debugger to the language service + +.PARAMETER tsScript +The path to a custom language service script to use, e.g. "C:\src\TypeScript\built\local\typescriptServices.js" +#> +Param( + [int]$vsVersion = 14, + [switch]$enableDevMode, + [string]$tsScript +) + +$vsRegKey = "HKCU:\Software\Microsoft\VisualStudio\${vsVersion}.0" +$tsRegKey = "${vsRegKey}\TypeScriptLanguageService" + +if($enableDevMode -ne $true -and $tsScript -eq ""){ + Throw "You must either enable language service debugging (-enableDevMode), set a custom script (-tsScript), or both" +} + +if(!(Test-Path $vsRegKey)){ + Throw "Visual Studio ${vsVersion} is not installed" +} +if(!(Test-Path $tsRegKey)){ + # Create the TypeScript subkey if it doesn't exist + New-Item -path $tsRegKey +} + +if($tsScript -ne ""){ + if(!(Test-Path $tsScript)){ + Throw "Could not locate the TypeScript language service script at ${tsScript}" + } + Set-ItemProperty -path $tsRegKey -name CustomTypeScriptServicesFileLocation -value "${tsScript}" + Write-Host "Enabled custom TypeScript language service at ${tsScript} for Dev${vsVersion}" +} +if($enableDevMode){ + Set-ItemProperty -path $tsRegKey -name EnableDevMode -value 1 + Write-Host "Enabled developer mode for Dev${vsVersion}" +} diff --git a/src/compiler/tsc.ts b/src/compiler/tsc.ts index 4bcd0856ca8..3f7e6362e19 100644 --- a/src/compiler/tsc.ts +++ b/src/compiler/tsc.ts @@ -332,7 +332,7 @@ module ts { var program = createProgram(fileNames, compilerOptions, compilerHost); var exitStatus = compileProgram(); - var end = start - new Date().getTime(); + var end = new Date().getTime() - start; if (compilerOptions.listFiles) { forEach(program.getSourceFiles(), file => { @@ -357,7 +357,7 @@ module ts { reportTimeStatistic("Bind time", ts.bindTime); reportTimeStatistic("Check time", ts.checkTime); reportTimeStatistic("Emit time", ts.emitTime); - reportTimeStatistic("Total time", start - end); + reportTimeStatistic("Total time", end); } return { program, exitStatus }; diff --git a/src/harness/harness.ts b/src/harness/harness.ts index 534bf85d119..86a68af1145 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -795,9 +795,12 @@ module Harness { } } - export function createSourceFileAndAssertInvariants(fileName: string, sourceText: string, languageVersion: ts.ScriptTarget) { - var result = ts.createSourceFile(fileName, sourceText, languageVersion, /*setParentNodes:*/ true); - Utils.assertInvariants(result, /*parent:*/ undefined); + export function createSourceFileAndAssertInvariants(fileName: string, sourceText: string, languageVersion: ts.ScriptTarget, assertInvariants = true) { + // Only set the parent nodes if we're asserting invariants. We don't need them otherwise. + var result = ts.createSourceFile(fileName, sourceText, languageVersion, /*setParentNodes:*/ assertInvariants); + if (assertInvariants) { + Utils.assertInvariants(result, /*parent:*/ undefined); + } return result; } @@ -805,7 +808,6 @@ module Harness { export var defaultLibSourceFile = createSourceFileAndAssertInvariants(defaultLibFileName, IO.readFile(libFolder + 'lib.core.d.ts'), /*languageVersion*/ ts.ScriptTarget.Latest); export var defaultES6LibSourceFile = createSourceFileAndAssertInvariants(defaultLibFileName, IO.readFile(libFolder + 'lib.core.es6.d.ts'), /*languageVersion*/ ts.ScriptTarget.Latest); - // Cache these between executions so we don't have to re-parse them for every test export var fourslashFileName = 'fourslash.ts'; export var fourslashSourceFile: ts.SourceFile; @@ -926,7 +928,8 @@ module Harness { settingsCallback?: (settings: ts.CompilerOptions) => void, options?: ts.CompilerOptions, // Current directory is needed for rwcRunner to be able to use currentDirectory defined in json file - currentDirectory?: string) { + currentDirectory?: string, + assertInvariants = true) { options = options || { noResolve: false }; options.target = options.target || ts.ScriptTarget.ES3; @@ -1074,7 +1077,7 @@ module Harness { var register = (file: { unitName: string; content: string; }) => { if (file.content !== undefined) { var fileName = ts.normalizeSlashes(file.unitName); - filemap[getCanonicalFileName(fileName)] = createSourceFileAndAssertInvariants(fileName, file.content, options.target); + filemap[getCanonicalFileName(fileName)] = createSourceFileAndAssertInvariants(fileName, file.content, options.target, assertInvariants); } }; inputFiles.forEach(register); diff --git a/src/harness/rwcRunner.ts b/src/harness/rwcRunner.ts index b66322f8390..c1c3251cf02 100644 --- a/src/harness/rwcRunner.ts +++ b/src/harness/rwcRunner.ts @@ -90,7 +90,8 @@ module RWC { /*settingsCallback*/ undefined, opts.options, // Since all Rwc json file specified current directory in its json file, we need to pass this information to compilerHost // so that when the host is asked for current directory, it should give the value from json rather than from process - currentDirectory); + currentDirectory, + /*assertInvariants:*/ false); }); function getHarnessCompilerInputUnit(fileName: string) { diff --git a/src/services/services.ts b/src/services/services.ts index a1191e71cd7..ae61a05b127 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1143,6 +1143,9 @@ module ts { InMultiLineCommentTrivia, InSingleQuoteStringLiteral, InDoubleQuoteStringLiteral, + InTemplateHeadOrNoSubstitutionTemplate, + InTemplateMiddleOrTail, + InTemplateSubstitutionPosition, } export enum TokenClass { @@ -1168,7 +1171,26 @@ module ts { } export interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** @@ -5617,6 +5639,28 @@ module ts { noRegexTable[SyntaxKind.TrueKeyword] = true; noRegexTable[SyntaxKind.FalseKeyword] = true; + // Just a stack of TemplateHeads and OpenCurlyBraces, used to perform rudimentary (inexact) + // classification on template strings. Because of the context free nature of templates, + // the only precise way to classify a template portion would be by propagating the stack across + // lines, just as we do with the end-of-line state. However, this is a burden for implementers, + // and the behavior is entirely subsumed by the syntactic classifier anyway, so we instead + // flatten any nesting when the template stack is non-empty and encode it in the end-of-line state. + // Situations in which this fails are + // 1) When template strings are nested across different lines: + // `hello ${ `world + // ` }` + // + // Where on the second line, you will get the closing of a template, + // a closing curly, and a new template. + // + // 2) When substitution expressions have curly braces and the curly brace falls on the next line: + // `hello ${ () => { + // return "world" } } ` + // + // Where on the second line, you will get the 'return' keyword, + // a string literal, and a template end consisting of '} } `'. + var templateStack: SyntaxKind[] = []; + function isAccessibilityModifier(kind: SyntaxKind) { switch (kind) { case SyntaxKind.PublicKeyword: @@ -5650,13 +5694,19 @@ module ts { // if there are more cases we want the classifier to be better at. return true; } - - // 'classifyKeywordsInGenerics' should be 'true' when a syntactic classifier is not present. - function getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult { + + // If there is a syntactic classifier ('syntacticClassifierAbsent' is false), + // we will be more conservative in order to avoid conflicting with the syntactic classifier. + function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult { var offset = 0; var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; + // Empty out the template stack for reuse. + while (templateStack.length > 0) { + templateStack.pop(); + } + // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. // @@ -5675,6 +5725,17 @@ module ts { text = "/*\n" + text; offset = 3; break; + case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate: + text = "`\n" + text; + offset = 2; + break; + case EndOfLineState.InTemplateMiddleOrTail: + text = "}\n" + text; + offset = 2; + // fallthrough + case EndOfLineState.InTemplateSubstitutionPosition: + templateStack.push(SyntaxKind.TemplateHead); + break; } scanner.setText(text); @@ -5740,12 +5801,45 @@ module ts { token === SyntaxKind.NumberKeyword || token === SyntaxKind.BooleanKeyword || token === SyntaxKind.SymbolKeyword) { - if (angleBracketStack > 0 && !classifyKeywordsInGenerics) { - // If it looks like we're could be in something generic, don't classify this - // as a keyword. We may just get overwritten by the syntactic classifier, - // causing a noisy experience for the user. - token = SyntaxKind.Identifier; - } + if (angleBracketStack > 0 && !syntacticClassifierAbsent) { + // If it looks like we're could be in something generic, don't classify this + // as a keyword. We may just get overwritten by the syntactic classifier, + // causing a noisy experience for the user. + token = SyntaxKind.Identifier; + } + } + else if (token === SyntaxKind.TemplateHead) { + templateStack.push(token); + } + else if (token === SyntaxKind.OpenBraceToken) { + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack.length > 0) { + templateStack.push(token); + } + } + else if (token === SyntaxKind.CloseBraceToken) { + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack.length > 0) { + var lastTemplateStackToken = lastOrUndefined(templateStack); + + if (lastTemplateStackToken === SyntaxKind.TemplateHead) { + token = scanner.reScanTemplateToken(); + + // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us. + if (token === SyntaxKind.TemplateTail) { + templateStack.pop(); + } + else { + Debug.assert(token === SyntaxKind.TemplateMiddle, "Should have been a template middle. Was " + token); + } + } + else { + Debug.assert(lastTemplateStackToken === SyntaxKind.OpenBraceToken, "Should have been an open brace. Was: " + token); + templateStack.pop(); + } + } } lastNonTriviaToken = token; @@ -5790,6 +5884,22 @@ module ts { result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; } } + else if (isTemplateLiteralKind(token)) { + if (scanner.isUnterminated()) { + if (token === SyntaxKind.TemplateTail) { + result.finalLexState = EndOfLineState.InTemplateMiddleOrTail; + } + else if (token === SyntaxKind.NoSubstitutionTemplateLiteral) { + result.finalLexState = EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate; + } + else { + Debug.fail("Only 'NoSubstitutionTemplateLiteral's and 'TemplateTail's can be unterminated; got SyntaxKind #" + token); + } + } + } + else if (templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { + result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition; + } } } @@ -5893,6 +6003,9 @@ module ts { return TokenClass.Whitespace; case SyntaxKind.Identifier: default: + if (isTemplateLiteralKind(token)) { + return TokenClass.StringLiteral; + } return TokenClass.Identifier; } } diff --git a/src/services/shims.ts b/src/services/shims.ts index 0f9f1a12cd1..a0eb1cb1ac4 100644 --- a/src/services/shims.ts +++ b/src/services/shims.ts @@ -165,7 +165,7 @@ module ts { } export interface ClassifierShim extends Shim { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): string; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string; } export interface CoreServicesShim extends Shim { diff --git a/src/services/signatureHelp.ts b/src/services/signatureHelp.ts index 426bd732882..8b7dea13ead 100644 --- a/src/services/signatureHelp.ts +++ b/src/services/signatureHelp.ts @@ -295,8 +295,8 @@ module ts.SignatureHelp { var tagExpression = templateExpression.parent; Debug.assert(templateExpression.kind === SyntaxKind.TemplateExpression); - // If we're just after a template tail, don't show signature help. - if (node.kind === SyntaxKind.TemplateTail && position >= node.getEnd() && !(node).isUnterminated) { + // If we're just after a template tail, don't show signature help. + if (node.kind === SyntaxKind.TemplateTail && !isInsideTemplateLiteral(node, position)) { return undefined; } diff --git a/tests/baselines/reference/APISample_compile.js b/tests/baselines/reference/APISample_compile.js index 59ab821c3b2..8afa0a5b939 100644 --- a/tests/baselines/reference/APISample_compile.js +++ b/tests/baselines/reference/APISample_compile.js @@ -1743,6 +1743,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1764,7 +1767,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_compile.types b/tests/baselines/reference/APISample_compile.types index 36c0976269f..e2ccc04a7c3 100644 --- a/tests/baselines/reference/APISample_compile.types +++ b/tests/baselines/reference/APISample_compile.types @@ -5545,6 +5545,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5600,12 +5609,31 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/baselines/reference/APISample_linter.js b/tests/baselines/reference/APISample_linter.js index 5004cb7a421..74ec089e0d0 100644 --- a/tests/baselines/reference/APISample_linter.js +++ b/tests/baselines/reference/APISample_linter.js @@ -1774,6 +1774,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1795,7 +1798,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_linter.types b/tests/baselines/reference/APISample_linter.types index d629b3fac93..3e750b03148 100644 --- a/tests/baselines/reference/APISample_linter.types +++ b/tests/baselines/reference/APISample_linter.types @@ -5689,6 +5689,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5744,12 +5753,31 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/baselines/reference/APISample_transform.js b/tests/baselines/reference/APISample_transform.js index 4022ef8cad0..22f50173a4f 100644 --- a/tests/baselines/reference/APISample_transform.js +++ b/tests/baselines/reference/APISample_transform.js @@ -1775,6 +1775,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1796,7 +1799,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_transform.types b/tests/baselines/reference/APISample_transform.types index c07f78b99c3..d2cfc6f633b 100644 --- a/tests/baselines/reference/APISample_transform.types +++ b/tests/baselines/reference/APISample_transform.types @@ -5641,6 +5641,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5696,12 +5705,31 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/baselines/reference/APISample_watcher.js b/tests/baselines/reference/APISample_watcher.js index 52eeb0e3499..158562aeaf8 100644 --- a/tests/baselines/reference/APISample_watcher.js +++ b/tests/baselines/reference/APISample_watcher.js @@ -1812,6 +1812,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1833,7 +1836,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_watcher.types b/tests/baselines/reference/APISample_watcher.types index ccf4fe54146..36bce4fa9f5 100644 --- a/tests/baselines/reference/APISample_watcher.types +++ b/tests/baselines/reference/APISample_watcher.types @@ -5814,6 +5814,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5869,12 +5878,31 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives2.ts b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives2.ts new file mode 100644 index 00000000000..e7a5078dcd0 --- /dev/null +++ b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives2.ts @@ -0,0 +1,11 @@ +/// + +//// function foo(strs, ...rest) { +//// } +//// +//// /*1*/fo/*2*/o /*3*/`abcd${0 + 1}abcd{1 + 1}`/*4*/ /*5*/ + +test.markers().forEach(m => { + goTo.position(m.position); + verify.not.signatureHelpPresent(); +}); \ No newline at end of file diff --git a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives3.ts b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives3.ts new file mode 100644 index 00000000000..c84b92fee4f --- /dev/null +++ b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives3.ts @@ -0,0 +1,11 @@ +/// + +//// function foo(strs, ...rest) { +//// } +//// +//// /*1*/fo/*2*/o /*3*/`abcd${0 + 1}abcd{1 + 1}abcd`/*4*/ /*5*/ + +test.markers().forEach(m => { + goTo.position(m.position); + verify.not.signatureHelpPresent(); +}); \ No newline at end of file diff --git a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives4.ts b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives4.ts new file mode 100644 index 00000000000..338a8dc65ca --- /dev/null +++ b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives4.ts @@ -0,0 +1,11 @@ +/// + +//// function foo(strs, ...rest) { +//// } +//// +//// /*1*/fo/*2*/o /*3*/``/*4*/ /*5*/ + +test.markers().forEach(m => { + goTo.position(m.position); + verify.not.signatureHelpPresent(); +}); \ No newline at end of file diff --git a/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts new file mode 100644 index 00000000000..02383df9267 --- /dev/null +++ b/tests/cases/fourslash/signatureHelpTaggedTemplatesNegatives5.ts @@ -0,0 +1,11 @@ +/// + +//// function foo(strs, ...rest) { +//// } +//// +//// /*1*/fo/*2*/o /*3*/`abcd`/*4*/ + +test.markers().forEach(m => { + goTo.position(m.position); + verify.not.signatureHelpPresent(); +}); \ No newline at end of file diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index 01149764305..c834f85806d 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -4,6 +4,7 @@ interface ClassificationEntry { value: any; classification: ts.TokenClass; + position?: number; } describe('Colorization', function () { @@ -23,16 +24,23 @@ describe('Colorization', function () { return undefined; } - function punctuation(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Punctuation }; } - function keyword(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Keyword }; } - function operator(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Operator }; } - function comment(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Comment }; } - function whitespace(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Whitespace }; } - function identifier(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Identifier }; } - function numberLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.NumberLiteral }; } - function stringLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.StringLiteral }; } - function regExpLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.RegExpLiteral }; } - function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined }; } + function punctuation(text: string, position?: number) { return createClassification(text, ts.TokenClass.Punctuation, position); } + function keyword(text: string, position?: number) { return createClassification(text, ts.TokenClass.Keyword, position); } + function operator(text: string, position?: number) { return createClassification(text, ts.TokenClass.Operator, position); } + function comment(text: string, position?: number) { return createClassification(text, ts.TokenClass.Comment, position); } + function whitespace(text: string, position?: number) { return createClassification(text, ts.TokenClass.Whitespace, position); } + function identifier(text: string, position?: number) { return createClassification(text, ts.TokenClass.Identifier, position); } + function numberLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.NumberLiteral, position); } + function stringLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.StringLiteral, position); } + function regExpLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.RegExpLiteral, position); } + function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined, position: 0 }; } + function createClassification(text: string, tokenClass: ts.TokenClass, position?: number): ClassificationEntry { + return { + value: text, + classification: tokenClass, + position: position, + }; + } function testLexicalClassification(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void { var result = classifier.getClassificationsForLine(text, initialEndOfLineState); @@ -44,7 +52,7 @@ describe('Colorization', function () { assert.equal(result.finalLexState, expectedEntry.value, "final endOfLineState does not match expected."); } else { - var actualEntryPosition = text.indexOf(expectedEntry.value); + var actualEntryPosition = expectedEntry.position !== undefined ? expectedEntry.position : text.indexOf(expectedEntry.value); assert(actualEntryPosition >= 0, "token: '" + expectedEntry.value + "' does not exit in text: '" + text + "'."); var actualEntry = getEntryAtPosistion(result, actualEntryPosition); @@ -254,6 +262,106 @@ describe('Colorization', function () { finalEndOfLineState(ts.EndOfLineState.Start)); }); + it("classifies a single line no substitution template string correctly", () => { + testLexicalClassification("`number number public string`", + ts.EndOfLineState.Start, + stringLiteral("`number number public string`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution parts of a template string correctly", () => { + testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`", + ts.EndOfLineState.Start, + stringLiteral("`number '${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}' string '${"), + stringLiteral("'hello'"), + stringLiteral("}'`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies an unterminated no substitution template string correctly", () => { + testLexicalClassification("`hello world", + ts.EndOfLineState.Start, + stringLiteral("`hello world"), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline no-substitution/head template", () => { + testLexicalClassification("...", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline template middle/end",() => { + testLexicalClassification("...", + ts.EndOfLineState.InTemplateMiddleOrTail, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateMiddleOrTail)); + }); + it("classifies a termination of a multiline template head", () => { + testLexicalClassification("...${", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...${"), + finalEndOfLineState(ts.EndOfLineState.InTemplateSubstitutionPosition)); + }); + it("classifies the termination of a multiline no substitution template", () => { + testLexicalClassification("...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies the substitution parts and middle/tail of a multiline template string", () => { + testLexicalClassification("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies a template middle and propagates the end of line state",() => { + testLexicalClassification("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution expressions with curly braces appropriately", () => { + var pos = 0; + var lastLength = 0; + + testLexicalClassification("...${ () => { } } ${ { x: `1` } }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral(track("...${"), pos), + punctuation(track(" ", "("), pos), + punctuation(track(")"), pos), + punctuation(track(" ", "=>"), pos), + punctuation(track(" ", "{"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "} ${"), pos), + punctuation(track(" ", "{"), pos), + identifier(track(" ", "x"), pos), + punctuation(track(":"), pos), + stringLiteral(track(" ", "`1`"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "}...`"), pos), + finalEndOfLineState(ts.EndOfLineState.Start)); + + // Adjusts 'pos' by accounting for the length of each portion of the string, + // but only return the last given string + function track(...vals: string[]): string { + for (var i = 0, n = vals.length; i < n; i++) { + pos += lastLength; + lastLength = vals[i].length; + } + return ts.lastOrUndefined(vals); + } + }); + it("classifies partially written generics correctly.", function () { testLexicalClassification("Foo