Microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2025-11-18 17:21:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(396): (Dis)allow Octal and Bad Escape Sequences in String and (Tagged) Template Literals (#51837)

Browse Source
This commit is contained in:
形素叢集・ぐらふぃーむ
2023-03-24 16:15:50 -07:00
committed by GitHub
parent 24373d80ec
commit c2dc2fd9b9
132 changed files with 4803 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/compiler/binder.ts
-10
View File
@@ -253,7 +253,6 @@ import {
nodeIsPresent,
NonNullChain,
NonNullExpression,
NumericLiteral,
objectAllocator,
ObjectLiteralExpression,
OptionalChain,
@@ -298,7 +297,6 @@ import {
TextRange,
ThisExpression,
ThrowStatement,
TokenFlags,
tokenToString,
tracing,
TracingNode,
@@ -2573,12 +2571,6 @@ function createBinder(): (file: SourceFile, options: CompilerOptions) => void {
}
}
function checkStrictModeNumericLiteral(node: NumericLiteral) {
if (languageVersion < ScriptTarget.ES5 && inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) {
file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode));
}
}
function checkStrictModePostfixUnaryExpression(node: PostfixUnaryExpression) {
// Grammar checking
// The identifier eval or arguments may not appear as the LeftHandSideExpression of an
@@ -2823,8 +2815,6 @@ function createBinder(): (file: SourceFile, options: CompilerOptions) => void {
return checkStrictModeCatchClause(node as CatchClause);
case SyntaxKind.DeleteExpression:
return checkStrictModeDeleteExpression(node as DeleteExpression);
case SyntaxKind.NumericLiteral:
return checkStrictModeNumericLiteral(node as NumericLiteral);
case SyntaxKind.PostfixUnaryExpression:
return checkStrictModePostfixUnaryExpression(node as PostfixUnaryExpression);
case SyntaxKind.PrefixUnaryExpression:
src/compiler/checker.ts
+1 -27
View File
@@ -465,7 +465,6 @@ import {
isCatchClause,
isCatchClauseVariableDeclarationOrBindingElement,
isCheckJsEnabledForFile,
isChildOfNodeWithKind,
isClassDeclaration,
isClassElement,
isClassExpression,
@@ -48531,33 +48530,8 @@ export function createTypeChecker(host: TypeCheckerHost): TypeChecker {
return false;
}
function checkGrammarNumericLiteral(node: NumericLiteral): boolean {
// Grammar checking
if (node.numericLiteralFlags & TokenFlags.Octal) {
let diagnosticMessage: DiagnosticMessage | undefined;
if (languageVersion >= ScriptTarget.ES5) {
diagnosticMessage = Diagnostics.Octal_literals_are_not_available_when_targeting_ECMAScript_5_and_higher_Use_the_syntax_0;
}
else if (isChildOfNodeWithKind(node, SyntaxKind.LiteralType)) {
diagnosticMessage = Diagnostics.Octal_literal_types_must_use_ES2015_syntax_Use_the_syntax_0;
}
else if (isChildOfNodeWithKind(node, SyntaxKind.EnumMember)) {
diagnosticMessage = Diagnostics.Octal_literals_are_not_allowed_in_enums_members_initializer_Use_the_syntax_0;
}
if (diagnosticMessage) {
const withMinus = isPrefixUnaryExpression(node.parent) && node.parent.operator === SyntaxKind.MinusToken;
const literal = (withMinus ? "-" : "") + "0o" + node.text;
return grammarErrorOnNode(withMinus ? node.parent : node, diagnosticMessage, literal);
}
}
function checkGrammarNumericLiteral(node: NumericLiteral) {
// Realism (size) checking
checkNumericLiteralValueSize(node);
return false;
}
function checkNumericLiteralValueSize(node: NumericLiteral) {
// We should test against `getTextOfNode(node)` rather than `node.text`, because `node.text` for large numeric literals can contain "."
// e.g. `node.text` for numeric literal `1100000000000000000000` is `1.1e21`.
const isFractional = getTextOfNode(node).indexOf(".") !== -1;
src/compiler/diagnosticMessages.json
+13 -13
View File
@@ -227,10 +227,6 @@
"category": "Error",
"code": 1084
},
"Octal literals are not available when targeting ECMAScript 5 and higher. Use the syntax '{0}'.": {
"category": "Error",
"code": 1085
},
"'{0}' modifier cannot appear on a constructor declaration.": {
"category": "Error",
"code": 1089
@@ -351,7 +347,7 @@
"category": "Error",
"code": 1120
},
"Octal literals are not allowed in strict mode.": {
"Octal literals are not allowed. Use the syntax '{0}'.": {
"category": "Error",
"code": 1121
},
@@ -1597,6 +1593,18 @@
"category": "Error",
"code": 1486
},
"Octal escape sequences are not allowed. Use the syntax '{0}'.": {
"category": "Error",
"code": 1487
},
"Escape sequence '{0}' is not allowed.": {
"category": "Error",
"code": 1488
},
"Decimals with leading zeros are not allowed.": {
"category": "Error",
"code": 1489
},
"The types of '{0}' are incompatible between these types.": {
"category": "Error",
@@ -6504,14 +6512,6 @@
"category": "Error",
"code": 8016
},
"Octal literal types must use ES2015 syntax. Use the syntax '{0}'.": {
"category": "Error",
"code": 8017
},
"Octal literals are not allowed in enums members initializer. Use the syntax '{0}'.": {
"category": "Error",
"code": 8018
},
"Report errors in .js files.": {
"category": "Message",
"code": 8019
src/compiler/emitter.ts
+6 -2
View File
@@ -419,6 +419,7 @@ import {
TemplateSpan,
TextRange,
ThrowStatement,
TokenFlags,
tokenToString,
tracing,
TransformationResult,
@@ -3049,9 +3050,12 @@ export function createPrinter(printerOptions: PrinterOptions = {}, handlers: Pri
if (isNumericLiteral(expression)) {
// check if numeric literal is a decimal literal that was originally written with a dot
const text = getLiteralTextOfNode(expression as LiteralExpression, /*neverAsciiEscape*/ true, /*jsxAttributeEscape*/ false);
// If he number will be printed verbatim and it doesn't already contain a dot, add one
// If the number will be printed verbatim and it doesn't already contain a dot or an exponent indicator, add one
// if the expression doesn't have any comments that will be emitted.
return !expression.numericLiteralFlags && !stringContains(text, tokenToString(SyntaxKind.DotToken)!);
return !(expression.numericLiteralFlags & TokenFlags.WithSpecifier)
&& !stringContains(text, tokenToString(SyntaxKind.DotToken)!)
&& !stringContains(text, String.fromCharCode(CharacterCodes.E))
&& !stringContains(text, String.fromCharCode(CharacterCodes.e));
}
else if (isAccessExpression(expression)) {
// check if constant enum value is integer
src/compiler/parser.ts
+10 -11
View File
@@ -2128,8 +2128,8 @@ namespace Parser {
parseErrorAt(range.pos, range.end, message, ...args);
}
function scanError(message: DiagnosticMessage, length: number): void {
parseErrorAtPosition(scanner.getTokenEnd(), length, message);
function scanError(message: DiagnosticMessage, length: number, arg0?: any): void {
parseErrorAtPosition(scanner.getTokenEnd(), length, message, arg0);
}
function getNodePos(): number {
@@ -2188,10 +2188,6 @@ namespace Parser {
return currentToken = scanner.reScanTemplateToken(isTaggedTemplate);
}
function reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind {
return currentToken = scanner.reScanTemplateHeadOrNoSubstitutionTemplate();
}
function reScanLessThanToken(): SyntaxKind {
return currentToken = scanner.reScanLessThanToken();
}
@@ -3636,8 +3632,8 @@ namespace Parser {
}
function parseTemplateHead(isTaggedTemplate: boolean): TemplateHead {
if (isTaggedTemplate) {
reScanTemplateHeadOrNoSubstitutionTemplate();
if (!isTaggedTemplate && scanner.getTokenFlags() & TokenFlags.IsInvalid) {
reScanTemplateToken(/*isTaggedTemplate*/ false);
}
const fragment = parseLiteralLikeNode(token());
Debug.assert(fragment.kind === SyntaxKind.TemplateHead, "Template head has wrong token kind");
@@ -3660,7 +3656,6 @@ namespace Parser {
const pos = getNodePos();
const node =
isTemplateLiteralKind(kind) ? factory.createTemplateLiteralLikeNode(kind, scanner.getTokenValue(), getTemplateLiteralRawText(kind), scanner.getTokenFlags() & TokenFlags.TemplateLiteralLikeFlags) :
// Octal literals are not allowed in strict mode or ES5
// Note that theoretically the following condition would hold true literals like 009,
// which is not octal. But because of how the scanner separates the tokens, we would
// never get a token like this. Instead, we would get 00 and 9 as two separate tokens.
@@ -6351,7 +6346,7 @@ namespace Parser {
tag,
typeArguments,
token() === SyntaxKind.NoSubstitutionTemplateLiteral ?
(reScanTemplateHeadOrNoSubstitutionTemplate(), parseLiteralNode() as NoSubstitutionTemplateLiteral) :
(reScanTemplateToken(/*isTaggedTemplate*/ true), parseLiteralNode() as NoSubstitutionTemplateLiteral) :
parseTemplateExpression(/*isTaggedTemplate*/ true)
);
if (questionDotToken || tag.flags & NodeFlags.OptionalChain) {
@@ -6451,10 +6446,14 @@ namespace Parser {
function parsePrimaryExpression(): PrimaryExpression {
switch (token()) {
case SyntaxKind.NoSubstitutionTemplateLiteral:
if (scanner.getTokenFlags() & TokenFlags.IsInvalid) {
reScanTemplateToken(/*isTaggedTemplate*/ false);
}
// falls through
case SyntaxKind.NumericLiteral:
case SyntaxKind.BigIntLiteral:
case SyntaxKind.StringLiteral:
case SyntaxKind.NoSubstitutionTemplateLiteral:
return parseLiteralNode();
case SyntaxKind.ThisKeyword:
case SyntaxKind.SuperKeyword:
src/compiler/program.ts
-1
View File
@@ -1317,7 +1317,6 @@ export const plainJSErrors: Set<number> = new Set([
Diagnostics.Invalid_use_of_0_Modules_are_automatically_in_strict_mode.code,
Diagnostics.Invalid_use_of_0_in_strict_mode.code,
Diagnostics.A_label_is_not_allowed_here.code,
Diagnostics.Octal_literals_are_not_allowed_in_strict_mode.code,
Diagnostics.with_statements_are_not_allowed_in_strict_mode.code,
// grammar errors
Diagnostics.A_break_statement_can_only_be_used_within_an_enclosing_iteration_or_switch_statement.code,
src/compiler/scanner.ts
+194 -88
View File
@@ -18,6 +18,7 @@ import {
LanguageVariant,
LineAndCharacter,
MapLike,
padLeft,
parsePseudoBigInt,
positionIsSynthesized,
PunctuationOrKeywordSyntaxKind,
@@ -28,7 +29,7 @@ import {
trimStringStart,
} from "./_namespaces/ts";
export type ErrorCallback = (message: DiagnosticMessage, length: number) => void;
export type ErrorCallback = (message: DiagnosticMessage, length: number, arg0?: any) => void;
/** @internal */
export function tokenIsIdentifierOrKeyword(token: SyntaxKind): boolean {
@@ -71,6 +72,7 @@ export interface Scanner {
reScanSlashToken(): SyntaxKind;
reScanAsteriskEqualsToken(): SyntaxKind;
reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind;
/** @deprecated use {@link reScanTemplateToken}(false) */
reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind;
scanJsxIdentifier(): SyntaxKind;
scanJsxAttributeValue(): SyntaxKind;
@@ -1067,12 +1069,12 @@ export function createScanner(languageVersion: ScriptTarget,
return scanner;
function error(message: DiagnosticMessage): void;
function error(message: DiagnosticMessage, errPos: number, length: number): void;
function error(message: DiagnosticMessage, errPos: number = pos, length?: number): void {
function error(message: DiagnosticMessage, errPos: number, length: number, arg0?: any): void;
function error(message: DiagnosticMessage, errPos: number = pos, length?: number, arg0?: any): void {
if (onError) {
const oldPos = pos;
pos = errPos;
onError(message, length || 0);
onError(message, length || 0, arg0);
pos = oldPos;
}
}
@@ -1091,11 +1093,14 @@ export function createScanner(languageVersion: ScriptTarget,
isPreviousTokenSeparator = true;
result += text.substring(start, pos);
}
else if (isPreviousTokenSeparator) {
error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1);
}
else {
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1);
tokenFlags |= TokenFlags.ContainsInvalidSeparator;
if (isPreviousTokenSeparator) {
error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1);
}
else {
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1);
}
}
pos++;
start = pos;
@@ -1110,14 +1115,69 @@ export function createScanner(languageVersion: ScriptTarget,
break;
}
if (text.charCodeAt(pos - 1) === CharacterCodes._) {
tokenFlags |= TokenFlags.ContainsInvalidSeparator;
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1);
}
return result + text.substring(start, pos);
}
// Extract from Section 12.9.3
// NumericLiteral ::=
// | DecimalLiteral
// | DecimalBigIntegerLiteral
// | NonDecimalIntegerLiteral 'n'?
// | LegacyOctalIntegerLiteral
// DecimalBigIntegerLiteral ::=
// | '0n'
// | [1-9] DecimalDigits? 'n'
// | [1-9] '_' DecimalDigits 'n'
// DecimalLiteral ::=
// | DecimalIntegerLiteral? '.' DecimalDigits? ExponentPart?
// | '.' DecimalDigits ExponentPart?
// | DecimalIntegerLiteral ExponentPart?
// DecimalIntegerLiteral ::=
// | '0'
// | [1-9] '_'? DecimalDigits
// | NonOctalDecimalIntegerLiteral
// LegacyOctalIntegerLiteral ::= '0' [0-7]+
// NonOctalDecimalIntegerLiteral ::= '0' [0-7]* [89] [0-9]*
function scanNumber(): { type: SyntaxKind, value: string } {
const start = pos;
const mainFragment = scanNumberFragment();
let start = pos;
let mainFragment: string;
if (text.charCodeAt(pos) === CharacterCodes._0) {
pos++;
if (text.charCodeAt(pos) === CharacterCodes._) {
tokenFlags |= TokenFlags.ContainsSeparator | TokenFlags.ContainsInvalidSeparator;
error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1);
// treat it as a normal number literal
pos--;
mainFragment = scanNumberFragment();
}
// Separators are not allowed in the below cases
else if (!scanDigits()) {
// NonOctalDecimalIntegerLiteral, emit error later
// Separators in decimal and exponent parts are still allowed according to the spec
tokenFlags |= TokenFlags.ContainsLeadingZero;
mainFragment = "" + +tokenValue;
}
else if (!tokenValue) {
// a single zero
mainFragment = "0";
}
else {
// LegacyOctalIntegerLiteral
tokenValue = "" + parseInt(tokenValue, 8);
tokenFlags |= TokenFlags.Octal;
const withMinus = token === SyntaxKind.MinusToken;
const literal = (withMinus ? "-" : "") + "0o" + (+tokenValue).toString(8);
if (withMinus) start--;
error(Diagnostics.Octal_literals_are_not_allowed_Use_the_syntax_0, start, pos - start, literal);
return { type: SyntaxKind.NumericLiteral, value: tokenValue };
}
}
else {
mainFragment = scanNumberFragment();
}
let decimalFragment: string | undefined;
let scientificFragment: string | undefined;
if (text.charCodeAt(pos) === CharacterCodes.dot) {
@@ -1153,6 +1213,12 @@ export function createScanner(languageVersion: ScriptTarget,
result = text.substring(start, end); // No need to use all the fragments; no _ removal needed
}
if (tokenFlags & TokenFlags.ContainsLeadingZero) {
error(Diagnostics.Decimals_with_leading_zeros_are_not_allowed, start, end - start);
// if a literal has a leading zero, it must not be bigint
return { type: SyntaxKind.NumericLiteral, value: "" + +result };
}
if (decimalFragment !== undefined || tokenFlags & TokenFlags.Scientific) {
checkForIdentifierStartAfterNumericLiteral(start, decimalFragment === undefined && !!(tokenFlags & TokenFlags.Scientific));
return {
@@ -1190,12 +1256,17 @@ export function createScanner(languageVersion: ScriptTarget,
}
}
function scanOctalDigits(): number {
function scanDigits(): boolean {
const start = pos;
while (isOctalDigit(text.charCodeAt(pos))) {
let isOctal = true;
while (isDigit(text.charCodeAt(pos))) {
if (!isOctalDigit(text.charCodeAt(pos))) {
isOctal = false;
}
pos++;
}
return +(text.substring(start, pos));
tokenValue = text.substring(start, pos);
return isOctal;
}
/**
@@ -1278,7 +1349,7 @@ export function createScanner(languageVersion: ScriptTarget,
}
if (ch === CharacterCodes.backslash && !jsxAttributeString) {
result += text.substring(start, pos);
result += scanEscapeSequence();
result += scanEscapeSequence(/*shouldEmitInvalidEscapeError*/ true);
start = pos;
continue;
}
@@ -1297,7 +1368,7 @@ export function createScanner(languageVersion: ScriptTarget,
* Sets the current 'tokenValue' and returns a NoSubstitutionTemplateLiteral or
* a literal component of a TemplateExpression.
*/
function scanTemplateAndSetTokenValue(isTaggedTemplate: boolean): SyntaxKind {
function scanTemplateAndSetTokenValue(shouldEmitInvalidEscapeError: boolean): SyntaxKind {
const startedWithBacktick = text.charCodeAt(pos) === CharacterCodes.backtick;
pos++;
@@ -1335,7 +1406,7 @@ export function createScanner(languageVersion: ScriptTarget,
// Escape character
if (currChar === CharacterCodes.backslash) {
contents += text.substring(start, pos);
contents += scanEscapeSequence(isTaggedTemplate);
contents += scanEscapeSequence(shouldEmitInvalidEscapeError);
start = pos;
continue;
}
@@ -1364,7 +1435,22 @@ export function createScanner(languageVersion: ScriptTarget,
return resultingToken;
}
function scanEscapeSequence(isTaggedTemplate?: boolean): string {
// Extract from Section A.1
// EscapeSequence ::
// | CharacterEscapeSequence
// | 0 (?![0-9])
// | LegacyOctalEscapeSequence
// | NonOctalDecimalEscapeSequence
// | HexEscapeSequence
// | UnicodeEscapeSequence
// LegacyOctalEscapeSequence ::=
// | '0' (?=[89])
// | [1-7] (?![0-7])
// | [0-3] [0-7] (?![0-7])
// | [4-7] [0-7]
// | [0-3] [0-7] [0-7]
// NonOctalDecimalEscapeSequence ::= [89]
function scanEscapeSequence(shouldEmitInvalidEscapeError?: boolean): string {
const start = pos;
pos++;
if (pos >= end) {
@@ -1375,13 +1461,47 @@ export function createScanner(languageVersion: ScriptTarget,
pos++;
switch (ch) {
case CharacterCodes._0:
// '\01'
if (isTaggedTemplate && pos < end && isDigit(text.charCodeAt(pos))) {
pos++;
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
// Although '0' preceding any digit is treated as LegacyOctalEscapeSequence,
// '\08' should separately be interpreted as '\0' + '8'.
if (pos >= end || !isDigit(text.charCodeAt(pos))) {
return "\0";
}
return "\0";
// '\01', '\011'
// falls through
case CharacterCodes._1:
case CharacterCodes._2:
case CharacterCodes._3:
// '\1', '\17', '\177'
if (pos < end && isOctalDigit(text.charCodeAt(pos))) {
pos++;
}
// '\17', '\177'
// falls through
case CharacterCodes._4:
case CharacterCodes._5:
case CharacterCodes._6:
case CharacterCodes._7:
// '\4', '\47' but not '\477'
if (pos < end && isOctalDigit(text.charCodeAt(pos))) {
pos++;
}
// '\47'
tokenFlags |= TokenFlags.ContainsInvalidEscape;
if (shouldEmitInvalidEscapeError) {
const code = parseInt(text.substring(start + 1, pos), 8);
error(Diagnostics.Octal_escape_sequences_are_not_allowed_Use_the_syntax_0, start, pos - start, "\\x" + padLeft(code.toString(16), 2, "0"));
return String.fromCharCode(code);
}
return text.substring(start, pos);
case CharacterCodes._8:
case CharacterCodes._9:
// the invalid '\8' and '\9'
tokenFlags |= TokenFlags.ContainsInvalidEscape;
if (shouldEmitInvalidEscapeError) {
error(Diagnostics.Escape_sequence_0_is_not_allowed, start, pos - start, text.substring(start, pos));
return String.fromCharCode(ch);
}
return text.substring(start, pos);
case CharacterCodes.b:
return "\b";
case CharacterCodes.t:
@@ -1399,62 +1519,70 @@ export function createScanner(languageVersion: ScriptTarget,
case CharacterCodes.doubleQuote:
return "\"";
case CharacterCodes.u:
if (isTaggedTemplate) {
// '\u' or '\u0' or '\u00' or '\u000'
for (let escapePos = pos; escapePos < pos + 4; escapePos++) {
if (escapePos < end && !isHexDigit(text.charCodeAt(escapePos)) && text.charCodeAt(escapePos) !== CharacterCodes.openBrace) {
pos = escapePos;
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
}
}
}
// '\u{DDDDDDDD}'
if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
// '\u{DDDDDDDD}'
pos++;
// '\u{'
if (isTaggedTemplate && !isHexDigit(text.charCodeAt(pos))) {
const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false);
const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1;
// '\u{Not Code Point' or '\u{CodePoint'
if (escapedValue < 0) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
if (shouldEmitInvalidEscapeError) {
error(Diagnostics.Hexadecimal_digit_expected);
}
return text.substring(start, pos);
}
if (isTaggedTemplate) {
const savePos = pos;
const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false);
const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1;
// '\u{Not Code Point' or '\u{CodePoint'
if (!isCodePoint(escapedValue) || text.charCodeAt(pos) !== CharacterCodes.closeBrace) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
}
else {
pos = savePos;
if (!isCodePoint(escapedValue)) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
if (shouldEmitInvalidEscapeError) {
error(Diagnostics.An_extended_Unicode_escape_value_must_be_between_0x0_and_0x10FFFF_inclusive);
}
return text.substring(start, pos);
}
if (pos >= end) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
if (shouldEmitInvalidEscapeError) {
error(Diagnostics.Unexpected_end_of_text);
}
return text.substring(start, pos);
}
if (text.charCodeAt(pos) !== CharacterCodes.closeBrace) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
if (shouldEmitInvalidEscapeError) {
error(Diagnostics.Unterminated_Unicode_escape_sequence);
}
return text.substring(start, pos);
}
pos++;
tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
return scanExtendedUnicodeEscape();
return utf16EncodeAsString(escapedValue);
}
tokenFlags |= TokenFlags.UnicodeEscape;
// '\uDDDD'
return scanHexadecimalEscape(/*numDigits*/ 4);
for (; pos < start + 6; pos++) {
if (!(pos < end && isHexDigit(text.charCodeAt(pos)))) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
if (shouldEmitInvalidEscapeError) {
error(Diagnostics.Hexadecimal_digit_expected);
}
return text.substring(start, pos);
}
}
tokenFlags |= TokenFlags.UnicodeEscape;
return String.fromCharCode(parseInt(text.substring(start + 2, pos), 16));
case CharacterCodes.x:
if (isTaggedTemplate) {
if (!isHexDigit(text.charCodeAt(pos))) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
return text.substring(start, pos);
}
else if (!isHexDigit(text.charCodeAt(pos + 1))) {
pos++;
// '\xDD'
for (; pos < start + 4; pos++) {
if (!(pos < end && isHexDigit(text.charCodeAt(pos)))) {
tokenFlags |= TokenFlags.ContainsInvalidEscape;
if (shouldEmitInvalidEscapeError) {
error(Diagnostics.Hexadecimal_digit_expected);
}
return text.substring(start, pos);
}
}
// '\xDD'
return scanHexadecimalEscape(/*numDigits*/ 2);
tokenFlags |= TokenFlags.HexEscape;
return String.fromCharCode(parseInt(text.substring(start + 2, pos), 16));
// when encountering a LineContinuation (i.e. a backslash and a line terminator sequence),
// the line terminator is interpreted to be "the empty code unit sequence".
@@ -1472,18 +1600,6 @@ export function createScanner(languageVersion: ScriptTarget,
}
}
function scanHexadecimalEscape(numDigits: number): string {
const escapedValue = scanExactNumberOfHexDigits(numDigits, /*canHaveSeparators*/ false);
if (escapedValue >= 0) {
return String.fromCharCode(escapedValue);
}
else {
error(Diagnostics.Hexadecimal_digit_expected);
return "";
}
}
function scanExtendedUnicodeEscape(): string {
const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false);
const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1;
@@ -1743,7 +1859,7 @@ export function createScanner(languageVersion: ScriptTarget,
tokenValue = scanString();
return token = SyntaxKind.StringLiteral;
case CharacterCodes.backtick:
return token = scanTemplateAndSetTokenValue(/*isTaggedTemplate*/ false);
return token = scanTemplateAndSetTokenValue(/*shouldEmitInvalidEscapeError*/ false);
case CharacterCodes.percent:
if (text.charCodeAt(pos + 1) === CharacterCodes.equals) {
return pos += 2, token = SyntaxKind.PercentEqualsToken;
@@ -1926,15 +2042,6 @@ export function createScanner(languageVersion: ScriptTarget,
tokenFlags |= TokenFlags.OctalSpecifier;
return token = checkBigIntSuffix();
}
// Try to parse as an octal
if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) {
tokenValue = "" + scanOctalDigits();
tokenFlags |= TokenFlags.Octal;
return token = SyntaxKind.NumericLiteral;
}
// This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero
// can only be followed by an octal digit, a dot, or the end of the number literal. However, we are being
// permissive and allowing decimal digits of the form 08* and 09* (which many browsers also do).
// falls through
case CharacterCodes._1:
case CharacterCodes._2:
@@ -2307,14 +2414,13 @@ export function createScanner(languageVersion: ScriptTarget,
* Unconditionally back up and scan a template expression portion.
*/
function reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind {
Debug.assert(token === SyntaxKind.CloseBraceToken, "'reScanTemplateToken' should only be called on a '}'");
pos = tokenStart;
return token = scanTemplateAndSetTokenValue(isTaggedTemplate);
return token = scanTemplateAndSetTokenValue(!isTaggedTemplate);
}
function reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind {
pos = tokenStart;
return token = scanTemplateAndSetTokenValue(/*isTaggedTemplate*/ true);
return token = scanTemplateAndSetTokenValue(/*shouldEmitInvalidEscapeError*/ true);
}
function reScanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind {
src/compiler/transformers/taggedTemplate.ts
+2 -1
View File
@@ -18,6 +18,7 @@ import {
TemplateLiteralLikeNode,
TemplateMiddle,
TemplateTail,
TokenFlags,
TransformationContext,
visitEachChild,
visitNode,
@@ -96,7 +97,7 @@ export function processTaggedTemplateExpression(
}
function createTemplateCooked(factory: NodeFactory, template: TemplateHead | TemplateMiddle | TemplateTail | NoSubstitutionTemplateLiteral) {
return template.templateFlags ? factory.createVoidZero() : factory.createStringLiteral(template.text);
return template.templateFlags! & TokenFlags.IsInvalid ? factory.createVoidZero() : factory.createStringLiteral(template.text);
}
/**
src/compiler/types.ts
+22 -10
View File
@@ -2790,24 +2790,36 @@ export const enum TokenFlags {
/** @internal */
Unterminated = 1 << 2,
/** @internal */
ExtendedUnicodeEscape = 1 << 3,
Scientific = 1 << 4, // e.g. `10e2`
Octal = 1 << 5, // e.g. `0777`
HexSpecifier = 1 << 6, // e.g. `0x00000000`
BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000`
OctalSpecifier = 1 << 8, // e.g. `0o777`
ExtendedUnicodeEscape = 1 << 3, // e.g. `\u{10ffff}`
Scientific = 1 << 4, // e.g. `10e2`
Octal = 1 << 5, // e.g. `0777`
HexSpecifier = 1 << 6, // e.g. `0x00000000`
BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000`
OctalSpecifier = 1 << 8, // e.g. `0o777`
/** @internal */
ContainsSeparator = 1 << 9, // e.g. `0b1100_0101`
ContainsSeparator = 1 << 9, // e.g. `0b1100_0101`
/** @internal */
UnicodeEscape = 1 << 10,
UnicodeEscape = 1 << 10, // e.g. `\u00a0`
/** @internal */
ContainsInvalidEscape = 1 << 11, // e.g. `\uhello`
/** @internal */
HexEscape = 1 << 12, // e.g. `\xa0`
/** @internal */
ContainsLeadingZero = 1 << 13, // e.g. `0888`
/** @internal */
ContainsInvalidSeparator = 1 << 14, // e.g. `0_1`
/** @internal */
BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier,
/** @internal */
NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinaryOrOctalSpecifier | ContainsSeparator,
WithSpecifier = HexSpecifier | BinaryOrOctalSpecifier,
/** @internal */
TemplateLiteralLikeFlags = ContainsInvalidEscape,
StringLiteralFlags = HexEscape | UnicodeEscape | ExtendedUnicodeEscape | ContainsInvalidEscape,
/** @internal */
NumericLiteralFlags = Scientific | Octal | ContainsLeadingZero | WithSpecifier | ContainsSeparator | ContainsInvalidSeparator,
/** @internal */
TemplateLiteralLikeFlags = HexEscape | UnicodeEscape | ExtendedUnicodeEscape | ContainsInvalidEscape,
/** @internal */
IsInvalid = Octal | ContainsLeadingZero | ContainsInvalidSeparator | ContainsInvalidEscape,
}
export interface NumericLiteral extends LiteralExpression, Declaration {
src/compiler/utilities.ts
+7 -2
View File
@@ -1700,8 +1700,13 @@ function canUseOriginalText(node: LiteralLikeNode, flags: GetLiteralTextFlags):
return false;
}
if (isNumericLiteral(node) && node.numericLiteralFlags & TokenFlags.ContainsSeparator) {
return !!(flags & GetLiteralTextFlags.AllowNumericSeparator);
if (isNumericLiteral(node)) {
if (node.numericLiteralFlags & TokenFlags.IsInvalid) {
return false;
}
if (node.numericLiteralFlags & TokenFlags.ContainsSeparator) {
return !!(flags & GetLiteralTextFlags.AllowNumericSeparator);
}
}
return !isBigIntLiteral(node);