Microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2025-11-18 17:21:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename scanner token methods (#53078)

Browse Source
This commit is contained in:
Nathan Shively-Sanders
2023-03-07 13:54:44 -08:00
committed by GitHub
parent 3236ac9336
commit ff1e08f859
12 changed files with 172 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/compiler/parser.ts
+52 -52
View File
@@ -1838,14 +1838,14 @@ namespace Parser {
speculationHelper(() => {
const savedContextFlags = contextFlags;
contextFlags |= NodeFlags.AwaitContext;
scanner.setTextPos(nextStatement.pos);
scanner.resetTokenState(nextStatement.pos);
nextToken();
while (token() !== SyntaxKind.EndOfFileToken) {
const startPos = scanner.getStartPos();
const startPos = scanner.getTokenFullStart();
const statement = parseListElement(ParsingContext.SourceElements, parseStatement);
statements.push(statement);
if (startPos === scanner.getStartPos()) {
if (startPos === scanner.getTokenFullStart()) {
nextToken();
}
@@ -2095,7 +2095,7 @@ namespace Parser {
}
function parseErrorAtCurrentToken(message: DiagnosticMessage, arg0?: any): DiagnosticWithDetachedLocation | undefined {
return parseErrorAt(scanner.getTokenPos(), scanner.getTextPos(), message, arg0);
return parseErrorAt(scanner.getTokenStart(), scanner.getTokenEnd(), message, arg0);
}
function parseErrorAtPosition(start: number, length: number, message: DiagnosticMessage, arg0?: any): DiagnosticWithDetachedLocation | undefined {
@@ -2122,11 +2122,11 @@ namespace Parser {
}
function scanError(message: DiagnosticMessage, length: number): void {
parseErrorAtPosition(scanner.getTextPos(), length, message);
parseErrorAtPosition(scanner.getTokenEnd(), length, message);
}
function getNodePos(): number {
return scanner.getStartPos();
return scanner.getTokenFullStart();
}
function hasPrecedingJSDocComment() {
@@ -2156,7 +2156,7 @@ namespace Parser {
// if the keyword had an escape
if (isKeyword(currentToken) && (scanner.hasUnicodeEscape() || scanner.hasExtendedUnicodeEscape())) {
// issue a parse error for the escape
parseErrorAt(scanner.getTokenPos(), scanner.getTextPos(), Diagnostics.Keywords_cannot_contain_escape_characters);
parseErrorAt(scanner.getTokenStart(), scanner.getTokenEnd(), Diagnostics.Keywords_cannot_contain_escape_characters);
}
return nextTokenWithoutCheck();
}
@@ -2344,7 +2344,7 @@ namespace Parser {
return;
case "is":
parseErrorAt(pos, scanner.getTextPos(), Diagnostics.A_type_predicate_is_only_allowed_in_return_type_position_for_functions_and_methods);
parseErrorAt(pos, scanner.getTokenStart(), Diagnostics.A_type_predicate_is_only_allowed_in_return_type_position_for_functions_and_methods);
return;
case "module":
@@ -2538,12 +2538,12 @@ namespace Parser {
function createNodeArray<T extends Node>(elements: T[], pos: number, end?: number, hasTrailingComma?: boolean): NodeArray<T> {
const array = factoryCreateNodeArray(elements, hasTrailingComma);
setTextRangePosEnd(array, pos, end ?? scanner.getStartPos());
setTextRangePosEnd(array, pos, end ?? scanner.getTokenFullStart());
return array;
}
function finishNode<T extends Node>(node: T, pos: number, end?: number): T {
setTextRangePosEnd(node, pos, end ?? scanner.getStartPos());
setTextRangePosEnd(node, pos, end ?? scanner.getTokenFullStart());
if (contextFlags) {
(node as Mutable<T>).flags |= contextFlags;
}
@@ -2563,7 +2563,7 @@ namespace Parser {
function createMissingNode<T extends Node>(kind: T["kind"], reportAtCurrentPosition: boolean, diagnosticMessage: DiagnosticMessage, arg0?: any): T;
function createMissingNode<T extends Node>(kind: T["kind"], reportAtCurrentPosition: boolean, diagnosticMessage?: DiagnosticMessage, arg0?: any): T {
if (reportAtCurrentPosition) {
parseErrorAtPosition(scanner.getStartPos(), 0, diagnosticMessage!, arg0);
parseErrorAtPosition(scanner.getTokenFullStart(), 0, diagnosticMessage!, arg0);
}
else if (diagnosticMessage) {
parseErrorAtCurrentToken(diagnosticMessage, arg0);
@@ -3052,7 +3052,7 @@ namespace Parser {
return undefined;
}
const node = syntaxCursor.currentNode(pos ?? scanner.getStartPos());
const node = syntaxCursor.currentNode(pos ?? scanner.getTokenFullStart());
// Can't reuse a missing node.
// Can't reuse a node that intersected the change range.
@@ -3094,7 +3094,7 @@ namespace Parser {
function consumeNode(node: Node) {
// Move the scanner so it is after the node we just consumed.
scanner.setTextPos(node.end);
scanner.resetTokenState(node.end);
nextToken();
return node;
}
@@ -3387,14 +3387,14 @@ namespace Parser {
let commaStart = -1; // Meaning the previous token was not a comma
while (true) {
if (isListElement(kind, /*inErrorRecovery*/ false)) {
const startPos = scanner.getStartPos();
const startPos = scanner.getTokenFullStart();
const result = parseListElement(kind, parseElement);
if (!result) {
parsingContext = saveParsingContext;
return undefined;
}
list.push(result);
commaStart = scanner.getTokenPos();
commaStart = scanner.getTokenStart();
if (parseOptional(SyntaxKind.CommaToken)) {
// No need to check for a zero length node since we know we parsed a comma
@@ -3418,7 +3418,7 @@ namespace Parser {
if (considerSemicolonAsDelimiter && token() === SyntaxKind.SemicolonToken && !scanner.hasPrecedingLineBreak()) {
nextToken();
}
if (startPos === scanner.getStartPos()) {
if (startPos === scanner.getTokenFullStart()) {
// What we're parsing isn't actually remotely recognizable as a element and we've consumed no tokens whatsoever
// Consume a token to advance the parser in some way and avoid an infinite loop
// This can happen when we're speculatively parsing parenthesized expressions which we think may be arrow functions,
@@ -4430,7 +4430,7 @@ namespace Parser {
function parseImportTypeAssertions(): ImportTypeAssertionContainer {
const pos = getNodePos();
const openBracePosition = scanner.getTokenPos();
const openBracePosition = scanner.getTokenStart();
parseExpected(SyntaxKind.OpenBraceToken);
const multiLine = scanner.hasPrecedingLineBreak();
parseExpected(SyntaxKind.AssertKeyword);
@@ -5259,7 +5259,7 @@ namespace Parser {
}
function parsePossibleParenthesizedArrowFunctionExpression(allowReturnTypeInArrowFunction: boolean): ArrowFunction | undefined {
const tokenPos = scanner.getTokenPos();
const tokenPos = scanner.getTokenStart();
if (notParenthesizedArrow?.has(tokenPos)) {
return undefined;
}
@@ -6513,7 +6513,7 @@ namespace Parser {
function parseArrayLiteralExpression(): ArrayLiteralExpression {
const pos = getNodePos();
const openBracketPosition = scanner.getTokenPos();
const openBracketPosition = scanner.getTokenStart();
const openBracketParsed = parseExpected(SyntaxKind.OpenBracketToken);
const multiLine = scanner.hasPrecedingLineBreak();
const elements = parseDelimitedList(ParsingContext.ArrayLiteralMembers, parseArgumentOrArrayLiteralElement);
@@ -6579,7 +6579,7 @@ namespace Parser {
function parseObjectLiteralExpression(): ObjectLiteralExpression {
const pos = getNodePos();
const openBracePosition = scanner.getTokenPos();
const openBracePosition = scanner.getTokenStart();
const openBraceParsed = parseExpected(SyntaxKind.OpenBraceToken);
const multiLine = scanner.hasPrecedingLineBreak();
const properties = parseDelimitedList(ParsingContext.ObjectLiteralMembers, parseObjectLiteralElement, /*considerSemicolonAsDelimiter*/ true);
@@ -6649,7 +6649,7 @@ namespace Parser {
function parseBlock(ignoreMissingOpenBrace: boolean, diagnosticMessage?: DiagnosticMessage): Block {
const pos = getNodePos();
const hasJSDoc = hasPrecedingJSDocComment();
const openBracePosition = scanner.getTokenPos();
const openBracePosition = scanner.getTokenStart();
const openBraceParsed = parseExpected(SyntaxKind.OpenBraceToken, diagnosticMessage);
if (openBraceParsed || ignoreMissingOpenBrace) {
const multiLine = scanner.hasPrecedingLineBreak();
@@ -6710,7 +6710,7 @@ namespace Parser {
const pos = getNodePos();
const hasJSDoc = hasPrecedingJSDocComment();
parseExpected(SyntaxKind.IfKeyword);
const openParenPosition = scanner.getTokenPos();
const openParenPosition = scanner.getTokenStart();
const openParenParsed = parseExpected(SyntaxKind.OpenParenToken);
const expression = allowInAnd(parseExpression);
parseExpectedMatchingBrackets(SyntaxKind.OpenParenToken, SyntaxKind.CloseParenToken, openParenParsed, openParenPosition);
@@ -6725,7 +6725,7 @@ namespace Parser {
parseExpected(SyntaxKind.DoKeyword);
const statement = parseStatement();
parseExpected(SyntaxKind.WhileKeyword);
const openParenPosition = scanner.getTokenPos();
const openParenPosition = scanner.getTokenStart();
const openParenParsed = parseExpected(SyntaxKind.OpenParenToken);
const expression = allowInAnd(parseExpression);
parseExpectedMatchingBrackets(SyntaxKind.OpenParenToken, SyntaxKind.CloseParenToken, openParenParsed, openParenPosition);
@@ -6742,7 +6742,7 @@ namespace Parser {
const pos = getNodePos();
const hasJSDoc = hasPrecedingJSDocComment();
parseExpected(SyntaxKind.WhileKeyword);
const openParenPosition = scanner.getTokenPos();
const openParenPosition = scanner.getTokenStart();
const openParenParsed = parseExpected(SyntaxKind.OpenParenToken);
const expression = allowInAnd(parseExpression);
parseExpectedMatchingBrackets(SyntaxKind.OpenParenToken, SyntaxKind.CloseParenToken, openParenParsed, openParenPosition);
@@ -6821,7 +6821,7 @@ namespace Parser {
const pos = getNodePos();
const hasJSDoc = hasPrecedingJSDocComment();
parseExpected(SyntaxKind.WithKeyword);
const openParenPosition = scanner.getTokenPos();
const openParenPosition = scanner.getTokenStart();
const openParenParsed = parseExpected(SyntaxKind.OpenParenToken);
const expression = allowInAnd(parseExpression);
parseExpectedMatchingBrackets(SyntaxKind.OpenParenToken, SyntaxKind.CloseParenToken, openParenParsed, openParenPosition);
@@ -8103,7 +8103,7 @@ namespace Parser {
function parseImportDeclarationOrImportEqualsDeclaration(pos: number, hasJSDoc: boolean, modifiers: NodeArray<ModifierLike> | undefined): ImportEqualsDeclaration | ImportDeclaration {
parseExpected(SyntaxKind.ImportKeyword);
const afterImportPos = scanner.getStartPos();
const afterImportPos = scanner.getTokenFullStart();
// We don't parse the identifier here in await context, instead we will report a grammar error in the checker.
let identifier: Identifier | undefined;
@@ -8160,7 +8160,7 @@ namespace Parser {
if (!skipAssertKeyword) {
parseExpected(SyntaxKind.AssertKeyword);
}
const openBracePosition = scanner.getTokenPos();
const openBracePosition = scanner.getTokenStart();
if (parseExpected(SyntaxKind.OpenBraceToken)) {
const multiLine = scanner.hasPrecedingLineBreak();
const elements = parseDelimitedList(ParsingContext.AssertEntries, parseAssertEntry, /*considerSemicolonAsDelimiter*/ true);
@@ -8295,8 +8295,8 @@ namespace Parser {
// IdentifierName
// IdentifierName as IdentifierName
let checkIdentifierIsKeyword = isKeyword(token()) && !isIdentifier();
let checkIdentifierStart = scanner.getTokenPos();
let checkIdentifierEnd = scanner.getTextPos();
let checkIdentifierStart = scanner.getTokenStart();
let checkIdentifierEnd = scanner.getTokenEnd();
let isTypeOnly = false;
let propertyName: Identifier | undefined;
let canParseAsKeyword = true;
@@ -8363,8 +8363,8 @@ namespace Parser {
function parseNameWithKeywordCheck() {
checkIdentifierIsKeyword = isKeyword(token()) && !isIdentifier();
checkIdentifierStart = scanner.getTokenPos();
checkIdentifierEnd = scanner.getTextPos();
checkIdentifierStart = scanner.getTokenStart();
checkIdentifierEnd = scanner.getTokenEnd();
return parseIdentifierName();
}
}
@@ -8649,8 +8649,8 @@ namespace Parser {
break loop;
case SyntaxKind.OpenBraceToken:
state = JSDocState.SavingComments;
const commentEnd = scanner.getStartPos();
const linkStart = scanner.getTextPos() - 1;
const commentEnd = scanner.getTokenFullStart();
const linkStart = scanner.getTokenEnd() - 1;
const link = parseJSDocLink(linkStart);
if (link) {
if (!linkEnd) {
@@ -8659,7 +8659,7 @@ namespace Parser {
parts.push(finishNode(factory.createJSDocText(comments.join("")), linkEnd ?? start, commentEnd));
parts.push(link);
comments = [];
linkEnd = scanner.getTextPos();
linkEnd = scanner.getTokenEnd();
break;
}
// fallthrough if it's not a {@link sequence
@@ -8745,7 +8745,7 @@ namespace Parser {
function parseTag(margin: number) {
Debug.assert(token() === SyntaxKind.AtToken);
const start = scanner.getTokenPos();
const start = scanner.getTokenStart();
nextTokenJSDoc();
const tagName = parseJSDocIdentifierName(/*message*/ undefined);
@@ -8878,7 +8878,7 @@ namespace Parser {
comments.push(scanner.getTokenText());
break;
}
scanner.setTextPos(scanner.getTextPos() - 1);
scanner.resetTokenState(scanner.getTokenEnd() - 1);
// falls through
case SyntaxKind.EndOfFileToken:
// Done
@@ -8898,14 +8898,14 @@ namespace Parser {
break;
case SyntaxKind.OpenBraceToken:
state = JSDocState.SavingComments;
const commentEnd = scanner.getStartPos();
const linkStart = scanner.getTextPos() - 1;
const commentEnd = scanner.getTokenFullStart();
const linkStart = scanner.getTokenEnd() - 1;
const link = parseJSDocLink(linkStart);
if (link) {
parts.push(finishNode(factory.createJSDocText(comments.join("")), linkEnd ?? commentsPos, commentEnd));
parts.push(link);
comments = [];
linkEnd = scanner.getTextPos();
linkEnd = scanner.getTokenEnd();
}
else {
pushComment(scanner.getTokenText());
@@ -8946,7 +8946,7 @@ namespace Parser {
if (comments.length) {
parts.push(finishNode(factory.createJSDocText(comments.join("")), linkEnd ?? commentsPos));
}
return createNodeArray(parts, commentsPos, scanner.getTextPos());
return createNodeArray(parts, commentsPos, scanner.getTokenEnd());
}
else if (comments.length) {
return comments.join("");
@@ -8985,7 +8985,7 @@ namespace Parser {
const create = linkType === "link" ? factory.createJSDocLink
: linkType === "linkcode" ? factory.createJSDocLinkCode
: factory.createJSDocLinkPlain;
return finishNode(create(name, text.join("")), start, scanner.getTextPos());
return finishNode(create(name, text.join("")), start, scanner.getTokenEnd());
}
function parseJSDocLinkPrefix() {
@@ -9105,7 +9105,7 @@ namespace Parser {
function parseReturnTag(start: number, tagName: Identifier, indent: number, indentText: string): JSDocReturnTag {
if (some(tags, isJSDocReturnTag)) {
parseErrorAt(tagName.pos, scanner.getTokenPos(), Diagnostics._0_tag_already_specified, tagName.escapedText);
parseErrorAt(tagName.pos, scanner.getTokenStart(), Diagnostics._0_tag_already_specified, tagName.escapedText);
}
const typeExpression = tryParseTypeExpression();
@@ -9114,7 +9114,7 @@ namespace Parser {
function parseTypeTag(start: number, tagName: Identifier, indent?: number, indentText?: string): JSDocTypeTag {
if (some(tags, isJSDocTypeTag)) {
parseErrorAt(tagName.pos, scanner.getTokenPos(), Diagnostics._0_tag_already_specified, tagName.escapedText);
parseErrorAt(tagName.pos, scanner.getTokenStart(), Diagnostics._0_tag_already_specified, tagName.escapedText);
}
const typeExpression = parseJSDocTypeExpression(/*mayOmitBraces*/ true);
@@ -9139,10 +9139,10 @@ namespace Parser {
function parseAuthorTag(start: number, tagName: Identifier, indent: number, indentText: string): JSDocAuthorTag {
const commentStart = getNodePos();
const textOnly = parseAuthorNameAndEmail();
let commentEnd = scanner.getStartPos();
let commentEnd = scanner.getTokenFullStart();
const comments = parseTrailingTagComments(start, commentEnd, indent, indentText);
if (!comments) {
commentEnd = scanner.getStartPos();
commentEnd = scanner.getTokenFullStart();
}
const allParts = typeof comments !== "string"
? createNodeArray(concatenate([finishNode(textOnly, commentStart, commentEnd)], comments) as JSDocComment[], commentStart) // cast away readonly
@@ -9163,7 +9163,7 @@ namespace Parser {
}
else if (token === SyntaxKind.GreaterThanToken && inEmail) {
comments.push(scanner.getTokenText());
scanner.setTextPos(scanner.getTokenPos() + 1);
scanner.resetTokenState(scanner.getTokenEnd());
break;
}
comments.push(scanner.getTokenText());
@@ -9284,7 +9284,7 @@ namespace Parser {
}
function parseJSDocTypeNameWithNamespace(nested?: boolean) {
const pos = scanner.getTokenPos();
const start = scanner.getTokenStart();
if (!tokenIsIdentifierOrKeyword(token())) {
return undefined;
}
@@ -9297,7 +9297,7 @@ namespace Parser {
body,
nested ? NodeFlags.NestedNamespace : undefined
) as JSDocNamespaceDeclaration;
return finishNode(jsDocNamespaceNode, pos);
return finishNode(jsDocNamespaceNode, start);
}
if (nested) {
@@ -9408,7 +9408,7 @@ namespace Parser {
function tryParseChildTag(target: PropertyLikeParse, indent: number): JSDocTypeTag | JSDocPropertyTag | JSDocParameterTag | false {
Debug.assert(token() === SyntaxKind.AtToken);
const start = scanner.getStartPos();
const start = scanner.getTokenFullStart();
nextTokenJSDoc();
const tagName = parseJSDocIdentifierName();
@@ -9520,11 +9520,11 @@ namespace Parser {
}
identifierCount++;
const pos = scanner.getTokenPos();
const end = scanner.getTextPos();
const start = scanner.getTokenStart();
const end = scanner.getTokenEnd();
const originalKeywordKind = token();
const text = internIdentifier(scanner.getTokenValue());
const result = finishNode(factoryCreateIdentifier(text, originalKeywordKind), pos, end);
const result = finishNode(factoryCreateIdentifier(text, originalKeywordKind), start, end);
nextTokenJSDoc();
return result;
}
src/compiler/scanner.ts
+55 -43
View File
@@ -40,9 +40,15 @@ export function tokenIsIdentifierOrKeywordOrGreaterThan(token: SyntaxKind): bool
}
export interface Scanner {
/** @deprecated use {@link getTokenFullStart} */
getStartPos(): number;
getToken(): SyntaxKind;
getTokenFullStart(): number;
getTokenStart(): number;
getTokenEnd(): number;
/** @deprecated use {@link getTokenEnd} */
getTextPos(): number;
/** @deprecated use {@link getTokenStart} */
getTokenPos(): number;
getTokenText(): string;
getTokenValue(): string;
@@ -86,7 +92,9 @@ export interface Scanner {
setOnError(onError: ErrorCallback | undefined): void;
setScriptTarget(scriptTarget: ScriptTarget): void;
setLanguageVariant(variant: LanguageVariant): void;
/** @deprecated use {@link resetTokenState} */
setTextPos(textPos: number): void;
resetTokenState(pos: number): void;
/** @internal */
setInJSDocType(inType: boolean): void;
// Invokes the provided callback then unconditionally restores the scanner to the state it
@@ -974,10 +982,10 @@ export function createScanner(languageVersion: ScriptTarget,
var end: number;
// Start position of whitespace before current token
var startPos: number;
var fullStartPos: number;
// Start position of text of current token
var tokenPos: number;
var tokenStart: number;
var token: SyntaxKind;
var tokenValue!: string;
@@ -989,11 +997,14 @@ export function createScanner(languageVersion: ScriptTarget,
setText(text, start, length);
var scanner: Scanner = {
getStartPos: () => startPos,
getTokenFullStart: () => fullStartPos,
getStartPos: () => fullStartPos,
getTokenEnd: () => pos,
getTextPos: () => pos,
getToken: () => token,
getTokenPos: () => tokenPos,
getTokenText: () => text.substring(tokenPos, pos),
getTokenStart: () => tokenStart,
getTokenPos: () => tokenStart,
getTokenText: () => text.substring(tokenStart, pos),
getTokenValue: () => tokenValue,
hasUnicodeEscape: () => (tokenFlags & TokenFlags.UnicodeEscape) !== 0,
hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0,
@@ -1027,7 +1038,8 @@ export function createScanner(languageVersion: ScriptTarget,
setScriptTarget,
setLanguageVariant,
setOnError,
setTextPos,
resetTokenState,
setTextPos: resetTokenState,
setInJSDocType,
tryScan,
lookAhead,
@@ -1039,7 +1051,7 @@ export function createScanner(languageVersion: ScriptTarget,
Object.defineProperty(scanner, "__debugShowCurrentPositionInText", {
get: () => {
const text = scanner.getText();
return text.slice(0, scanner.getStartPos()) + "║" + text.slice(scanner.getStartPos());
return text.slice(0, scanner.getTokenFullStart()) + "║" + text.slice(scanner.getTokenFullStart());
},
});
}
@@ -1638,11 +1650,11 @@ export function createScanner(languageVersion: ScriptTarget,
}
function scan(): SyntaxKind {
startPos = pos;
fullStartPos = pos;
tokenFlags = TokenFlags.None;
let asteriskSeen = false;
while (true) {
tokenPos = pos;
tokenStart = pos;
if (pos >= end) {
return token = SyntaxKind.EndOfFileToken;
}
@@ -1810,9 +1822,9 @@ export function createScanner(languageVersion: ScriptTarget,
commentDirectives = appendIfCommentDirective(
commentDirectives,
text.slice(tokenPos, pos),
text.slice(tokenStart, pos),
commentDirectiveRegExSingleLine,
tokenPos,
tokenStart,
);
if (skipTrivia) {
@@ -1830,7 +1842,7 @@ export function createScanner(languageVersion: ScriptTarget,
}
let commentClosed = false;
let lastLineStart = tokenPos;
let lastLineStart = tokenStart;
while (pos < end) {
const ch = text.charCodeAt(pos);
@@ -2139,7 +2151,7 @@ export function createScanner(languageVersion: ScriptTarget,
function reScanInvalidIdentifier(): SyntaxKind {
Debug.assert(token === SyntaxKind.Unknown, "'reScanInvalidIdentifier' should only be called when the current token is 'SyntaxKind.Unknown'.");
pos = tokenPos = startPos;
pos = tokenStart = fullStartPos;
tokenFlags = 0;
const ch = codePointAt(text, pos);
const identifierKind = scanIdentifier(ch, ScriptTarget.ESNext);
@@ -2155,7 +2167,7 @@ export function createScanner(languageVersion: ScriptTarget,
if (isIdentifierStart(ch, languageVersion)) {
pos += charSize(ch);
while (pos < end && isIdentifierPart(ch = codePointAt(text, pos), languageVersion)) pos += charSize(ch);
tokenValue = text.substring(tokenPos, pos);
tokenValue = text.substring(tokenStart, pos);
if (ch === CharacterCodes.backslash) {
tokenValue += scanIdentifierParts();
}
@@ -2188,13 +2200,13 @@ export function createScanner(languageVersion: ScriptTarget,
function reScanAsteriskEqualsToken(): SyntaxKind {
Debug.assert(token === SyntaxKind.AsteriskEqualsToken, "'reScanAsteriskEqualsToken' should only be called on a '*='");
pos = tokenPos + 1;
pos = tokenStart + 1;
return token = SyntaxKind.EqualsToken;
}
function reScanSlashToken(): SyntaxKind {
if (token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) {
let p = tokenPos + 1;
let p = tokenStart + 1;
let inEscape = false;
let inCharacterClass = false;
while (true) {
@@ -2240,7 +2252,7 @@ export function createScanner(languageVersion: ScriptTarget,
p++;
}
pos = p;
tokenValue = text.substring(tokenPos, pos);
tokenValue = text.substring(tokenStart, pos);
token = SyntaxKind.RegularExpressionLiteral;
}
return token;
@@ -2288,23 +2300,23 @@ export function createScanner(languageVersion: ScriptTarget,
*/
function reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind {
Debug.assert(token === SyntaxKind.CloseBraceToken, "'reScanTemplateToken' should only be called on a '}'");
pos = tokenPos;
pos = tokenStart;
return token = scanTemplateAndSetTokenValue(isTaggedTemplate);
}
function reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind {
pos = tokenPos;
pos = tokenStart;
return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ true);
}
function reScanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind {
pos = tokenPos = startPos;
pos = tokenStart = fullStartPos;
return token = scanJsxToken(allowMultilineJsxText);
}
function reScanLessThanToken(): SyntaxKind {
if (token === SyntaxKind.LessThanLessThanToken) {
pos = tokenPos + 1;
pos = tokenStart + 1;
return token = SyntaxKind.LessThanToken;
}
return token;
@@ -2312,7 +2324,7 @@ export function createScanner(languageVersion: ScriptTarget,
function reScanHashToken(): SyntaxKind {
if (token === SyntaxKind.PrivateIdentifier) {
pos = tokenPos + 1;
pos = tokenStart + 1;
return token = SyntaxKind.HashToken;
}
return token;
@@ -2320,12 +2332,12 @@ export function createScanner(languageVersion: ScriptTarget,
function reScanQuestionToken(): SyntaxKind {
Debug.assert(token === SyntaxKind.QuestionQuestionToken, "'reScanQuestionToken' should only be called on a '??'");
pos = tokenPos + 1;
pos = tokenStart + 1;
return token = SyntaxKind.QuestionToken;
}
function scanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind {
startPos = tokenPos = pos;
fullStartPos = tokenStart = pos;
if (pos >= end) {
return token = SyntaxKind.EndOfFileToken;
@@ -2392,7 +2404,7 @@ export function createScanner(languageVersion: ScriptTarget,
pos++;
}
tokenValue = text.substring(startPos, pos);
tokenValue = text.substring(fullStartPos, pos);
return firstNonWhitespace === -1 ? SyntaxKind.JsxTextAllWhiteSpaces : SyntaxKind.JsxText;
}
@@ -2437,7 +2449,7 @@ export function createScanner(languageVersion: ScriptTarget,
}
function scanJsxAttributeValue(): SyntaxKind {
startPos = pos;
fullStartPos = pos;
switch (text.charCodeAt(pos)) {
case CharacterCodes.doubleQuote:
@@ -2451,12 +2463,12 @@ export function createScanner(languageVersion: ScriptTarget,
}
function reScanJsxAttributeValue(): SyntaxKind {
pos = tokenPos = startPos;
pos = tokenStart = fullStartPos;
return scanJsxAttributeValue();
}
function scanJsDocToken(): JSDocSyntaxKind {
startPos = tokenPos = pos;
fullStartPos = tokenStart = pos;
tokenFlags = TokenFlags.None;
if (pos >= end) {
return token = SyntaxKind.EndOfFileToken;
@@ -2531,7 +2543,7 @@ export function createScanner(languageVersion: ScriptTarget,
if (isIdentifierStart(ch, languageVersion)) {
let char = ch;
while (pos < end && isIdentifierPart(char = codePointAt(text, pos), languageVersion) || text.charCodeAt(pos) === CharacterCodes.minus) pos += charSize(char);
tokenValue = text.substring(tokenPos, pos);
tokenValue = text.substring(tokenStart, pos);
if (char === CharacterCodes.backslash) {
tokenValue += scanIdentifierParts();
}
@@ -2544,8 +2556,8 @@ export function createScanner(languageVersion: ScriptTarget,
function speculationHelper<T>(callback: () => T, isLookahead: boolean): T {
const savePos = pos;
const saveStartPos = startPos;
const saveTokenPos = tokenPos;
const saveStartPos = fullStartPos;
const saveTokenPos = tokenStart;
const saveToken = token;
const saveTokenValue = tokenValue;
const saveTokenFlags = tokenFlags;
@@ -2555,8 +2567,8 @@ export function createScanner(languageVersion: ScriptTarget,
// then unconditionally restore us to where we were.
if (!result || isLookahead) {
pos = savePos;
startPos = saveStartPos;
tokenPos = saveTokenPos;
fullStartPos = saveStartPos;
tokenStart = saveTokenPos;
token = saveToken;
tokenValue = saveTokenValue;
tokenFlags = saveTokenFlags;
@@ -2567,8 +2579,8 @@ export function createScanner(languageVersion: ScriptTarget,
function scanRange<T>(start: number, length: number, callback: () => T): T {
const saveEnd = end;
const savePos = pos;
const saveStartPos = startPos;
const saveTokenPos = tokenPos;
const saveStartPos = fullStartPos;
const saveTokenPos = tokenStart;
const saveToken = token;
const saveTokenValue = tokenValue;
const saveTokenFlags = tokenFlags;
@@ -2579,8 +2591,8 @@ export function createScanner(languageVersion: ScriptTarget,
end = saveEnd;
pos = savePos;
startPos = saveStartPos;
tokenPos = saveTokenPos;
fullStartPos = saveStartPos;
tokenStart = saveTokenPos;
token = saveToken;
tokenValue = saveTokenValue;
tokenFlags = saveTokenFlags;
@@ -2608,7 +2620,7 @@ export function createScanner(languageVersion: ScriptTarget,
function setText(newText: string | undefined, start: number | undefined, length: number | undefined) {
text = newText || "";
end = length === undefined ? text.length : start! + length;
setTextPos(start || 0);
resetTokenState(start || 0);
}
function setOnError(errorCallback: ErrorCallback | undefined) {
@@ -2623,11 +2635,11 @@ export function createScanner(languageVersion: ScriptTarget,
languageVariant = variant;
}
function setTextPos(textPos: number) {
Debug.assert(textPos >= 0);
pos = textPos;
startPos = textPos;
tokenPos = textPos;
function resetTokenState(position: number) {
Debug.assert(position >= 0);
pos = position;
fullStartPos = position;
tokenStart = position;
token = SyntaxKind.Unknown;
tokenValue = undefined!;
tokenFlags = TokenFlags.None;
src/compiler/utilities.ts
+3 -3
View File
@@ -2150,8 +2150,8 @@ export function createDiagnosticForRange(sourceFile: SourceFile, range: TextRang
export function getSpanOfTokenAtPosition(sourceFile: SourceFile, pos: number): TextSpan {
const scanner = createScanner(sourceFile.languageVersion, /*skipTrivia*/ true, sourceFile.languageVariant, sourceFile.text, /*onError:*/ undefined, pos);
scanner.scan();
const start = scanner.getTokenPos();
return createTextSpanFromBounds(start, scanner.getTextPos());
const start = scanner.getTokenStart();
return createTextSpanFromBounds(start, scanner.getTokenEnd());
}
/** @internal */
@@ -9558,7 +9558,7 @@ export function isValidBigIntString(s: string, roundTripOnly: boolean): boolean
// * a bigint can be scanned, and that when it is scanned, it is
// * the full length of the input string (so the scanner is one character beyond the augmented input length)
// * it does not contain a numeric seperator (the `BigInt` constructor does not accept a numeric seperator in its input)
return success && result === SyntaxKind.BigIntLiteral && scanner.getTextPos() === (s.length + 1) && !(flags & TokenFlags.ContainsSeparator)
return success && result === SyntaxKind.BigIntLiteral && scanner.getTokenEnd() === (s.length + 1) && !(flags & TokenFlags.ContainsSeparator)
&& (!roundTripOnly || s === pseudoBigIntToString({ negative, base10Value: parsePseudoBigInt(scanner.getTokenValue()) }));
}
src/harness/fourslashImpl.ts
+4 -4
View File
@@ -4897,11 +4897,11 @@ function codeFence(code: string, lang?: string) {
function getRangeOfIdentifierTouchingPosition(content: string, position: number): ts.TextRange | undefined {
const scanner = ts.createScanner(ts.ScriptTarget.Latest, /*skipTrivia*/ true, ts.LanguageVariant.Standard, content);
while (scanner.scan() !== ts.SyntaxKind.EndOfFileToken) {
const tokenStart = scanner.getStartPos();
if (scanner.getToken() === ts.SyntaxKind.Identifier && tokenStart <= position && scanner.getTextPos() >= position) {
return { pos: tokenStart, end: scanner.getTextPos() };
const tokenFullStart = scanner.getTokenFullStart();
if (scanner.getToken() === ts.SyntaxKind.Identifier && tokenFullStart <= position && scanner.getTokenEnd() >= position) {
return { pos: tokenFullStart, end: scanner.getTokenEnd() };
}
if (tokenStart > position) {
if (tokenFullStart > position) {
break;
}
}
src/services/classifier.ts
+10 -10
View File
@@ -152,8 +152,8 @@ export function createClassifier(): Classifier {
handleToken();
lastNonTriviaToken = token;
}
const end = scanner.getTextPos();
pushEncodedClassification(scanner.getTokenPos(), end, offset, classFromKind(token), spans);
const end = scanner.getTokenEnd();
pushEncodedClassification(scanner.getTokenStart(), end, offset, classFromKind(token), spans);
if (end >= text.length) {
const end = getNewEndOfLineState(scanner, token, lastOrUndefined(templateStack));
if (end !== undefined) {
@@ -700,16 +700,16 @@ export function getEncodedSyntacticClassifications(cancellationToken: Cancellati
}
function classifyLeadingTriviaAndGetTokenStart(token: Node): number {
triviaScanner.setTextPos(token.pos);
triviaScanner.resetTokenState(token.pos);
while (true) {
const start = triviaScanner.getTextPos();
const start = triviaScanner.getTokenEnd();
// only bother scanning if we have something that could be trivia.
if (!couldStartTrivia(sourceFile.text, start)) {
return start;
}
const kind = triviaScanner.scan();
const end = triviaScanner.getTextPos();
const end = triviaScanner.getTokenEnd();
const width = end - start;
// The moment we get something that isn't trivia, then stop processing.
@@ -731,7 +731,7 @@ export function getEncodedSyntacticClassifications(cancellationToken: Cancellati
// Classifying a comment might cause us to reuse the trivia scanner
// (because of jsdoc comments). So after we classify the comment make
// sure we set the scanner position back to where it needs to be.
triviaScanner.setTextPos(end);
triviaScanner.resetTokenState(end);
continue;
case SyntaxKind.ConflictMarkerTrivia:
@@ -989,17 +989,17 @@ export function getEncodedSyntacticClassifications(cancellationToken: Cancellati
}
}
pushClassification(start, i - start, ClassificationType.comment);
mergeConflictScanner.setTextPos(i);
mergeConflictScanner.resetTokenState(i);
while (mergeConflictScanner.getTextPos() < end) {
while (mergeConflictScanner.getTokenEnd() < end) {
classifyDisabledCodeToken();
}
}
function classifyDisabledCodeToken() {
const start = mergeConflictScanner.getTextPos();
const start = mergeConflictScanner.getTokenEnd();
const tokenKind = mergeConflictScanner.scan();
const end = mergeConflictScanner.getTextPos();
const end = mergeConflictScanner.getTokenEnd();
const type = classifyTokenType(tokenKind);
if (type) {
src/services/formatting/formatting.ts
+6 -6
View File
@@ -529,7 +529,7 @@ function formatSpanWorker(
}
}
if (previousRange! && formattingScanner.getStartPos() >= originalRange.end) {
if (previousRange! && formattingScanner.getTokenFullStart() >= originalRange.end) {
// Formatting edits happen by looking at pairs of contiguous tokens (see `processPair`),
// typically inserting or deleting whitespace between them. The recursive `processNode`
// logic above bails out as soon as it encounters a token that is beyond the end of the
@@ -785,7 +785,7 @@ function formatSpanWorker(
});
// proceed any tokens in the node that are located after child nodes
while (formattingScanner.isOnToken() && formattingScanner.getStartPos() < originalRange.end) {
while (formattingScanner.isOnToken() && formattingScanner.getTokenFullStart() < originalRange.end) {
const tokenInfo = formattingScanner.readTokenInfo(node);
if (tokenInfo.token.end > Math.min(node.end, originalRange.end)) {
break;
@@ -839,7 +839,7 @@ function formatSpanWorker(
return inheritedIndentation;
}
while (formattingScanner.isOnToken() && formattingScanner.getStartPos() < originalRange.end) {
while (formattingScanner.isOnToken() && formattingScanner.getTokenFullStart() < originalRange.end) {
// proceed any parent tokens that are located prior to child.getStart()
const tokenInfo = formattingScanner.readTokenInfo(node);
if (tokenInfo.token.end > originalRange.end) {
@@ -856,7 +856,7 @@ function formatSpanWorker(
consumeTokenAndAdvanceScanner(tokenInfo, node, parentDynamicIndentation, node);
}
if (!formattingScanner.isOnToken() || formattingScanner.getStartPos() >= originalRange.end) {
if (!formattingScanner.isOnToken() || formattingScanner.getTokenFullStart() >= originalRange.end) {
return inheritedIndentation;
}
@@ -906,7 +906,7 @@ function formatSpanWorker(
if (listStartToken !== SyntaxKind.Unknown) {
// introduce a new indentation scope for lists (including list start and end tokens)
while (formattingScanner.isOnToken() && formattingScanner.getStartPos() < originalRange.end) {
while (formattingScanner.isOnToken() && formattingScanner.getTokenFullStart() < originalRange.end) {
const tokenInfo = formattingScanner.readTokenInfo(parent);
if (tokenInfo.token.end > nodes.pos) {
// stop when formatting scanner moves past the beginning of node list
@@ -947,7 +947,7 @@ function formatSpanWorker(
}
const listEndToken = getCloseTokenForOpenToken(listStartToken);
if (listEndToken !== SyntaxKind.Unknown && formattingScanner.isOnToken() && formattingScanner.getStartPos() < originalRange.end) {
if (listEndToken !== SyntaxKind.Unknown && formattingScanner.isOnToken() && formattingScanner.getTokenFullStart() < originalRange.end) {
let tokenInfo: TokenInfo | undefined = formattingScanner.readTokenInfo(parent);
if (tokenInfo.token.kind === SyntaxKind.CommaToken) {
// consume the comma
src/services/formatting/formattingScanner.ts
+22 -19
View File
@@ -28,6 +28,8 @@ const jsxScanner = createScanner(ScriptTarget.Latest, /*skipTrivia*/ false, Lang
/** @internal */
export interface FormattingScanner {
advance(): void;
getTokenFullStart(): number;
/** @deprecated use {@link getTokenFullStart} */
getStartPos(): number;
isOnToken(): boolean;
isOnEOF(): boolean;
@@ -54,7 +56,7 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
const scanner = languageVariant === LanguageVariant.JSX ? jsxScanner : standardScanner;
scanner.setText(text);
scanner.setTextPos(startPos);
scanner.resetTokenState(startPos);
let wasNewLine = true;
let leadingTrivia: TextRangeWithTriviaKind[] | undefined;
@@ -74,7 +76,8 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
lastTrailingTriviaWasNewLine: () => wasNewLine,
skipToEndOf,
skipToStartOf,
getStartPos: () => lastTokenInfo?.token.pos ?? scanner.getTokenPos(),
getTokenFullStart: () => lastTokenInfo?.token.pos ?? scanner.getTokenStart(),
getStartPos: () => lastTokenInfo?.token.pos ?? scanner.getTokenStart(),
});
lastTokenInfo = undefined;
@@ -84,7 +87,7 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
function advance(): void {
lastTokenInfo = undefined;
const isStarted = scanner.getStartPos() !== startPos;
const isStarted = scanner.getTokenFullStart() !== startPos;
if (isStarted) {
wasNewLine = !!trailingTrivia && last(trailingTrivia).kind === SyntaxKind.NewLineTrivia;
@@ -96,7 +99,7 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
leadingTrivia = undefined;
trailingTrivia = undefined;
let pos = scanner.getStartPos();
let pos = scanner.getTokenFullStart();
// Read leading trivia and token
while (pos < endPos) {
@@ -109,16 +112,16 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
scanner.scan();
const item: TextRangeWithTriviaKind = {
pos,
end: scanner.getStartPos(),
end: scanner.getTokenFullStart(),
kind: t
};
pos = scanner.getStartPos();
pos = scanner.getTokenFullStart();
leadingTrivia = append(leadingTrivia, item);
}
savedPos = scanner.getStartPos();
savedPos = scanner.getTokenFullStart();
}
function shouldRescanGreaterThanToken(node: Node): boolean {
@@ -193,18 +196,18 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
return fixTokenKind(lastTokenInfo, n);
}
if (scanner.getStartPos() !== savedPos) {
if (scanner.getTokenFullStart() !== savedPos) {
Debug.assert(lastTokenInfo !== undefined);
// readTokenInfo was called before but scan action differs - rescan text
scanner.setTextPos(savedPos);
scanner.resetTokenState(savedPos);
scanner.scan();
}
let currentToken = getNextToken(n, expectedScanAction);
const token = createTextRangeWithKind(
scanner.getStartPos(),
scanner.getTextPos(),
scanner.getTokenFullStart(),
scanner.getTokenEnd(),
currentToken,
);
@@ -212,14 +215,14 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
if (trailingTrivia) {
trailingTrivia = undefined;
}
while (scanner.getStartPos() < endPos) {
while (scanner.getTokenFullStart() < endPos) {
currentToken = scanner.scan();
if (!isTrivia(currentToken)) {
break;
}
const trivia = createTextRangeWithKind(
scanner.getStartPos(),
scanner.getTextPos(),
scanner.getTokenFullStart(),
scanner.getTokenEnd(),
currentToken,
);
@@ -286,7 +289,7 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
function readEOFTokenRange(): TextRangeWithKind<SyntaxKind.EndOfFileToken> {
Debug.assert(isOnEOF());
return createTextRangeWithKind(scanner.getStartPos(), scanner.getTextPos(), SyntaxKind.EndOfFileToken);
return createTextRangeWithKind(scanner.getTokenFullStart(), scanner.getTokenEnd(), SyntaxKind.EndOfFileToken);
}
function isOnToken(): boolean {
@@ -311,8 +314,8 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
}
function skipToEndOf(node: Node | NodeArray<Node>): void {
scanner.setTextPos(node.end);
savedPos = scanner.getStartPos();
scanner.resetTokenState(node.end);
savedPos = scanner.getTokenFullStart();
lastScanAction = undefined;
lastTokenInfo = undefined;
wasNewLine = false;
@@ -321,8 +324,8 @@ export function getFormattingScanner<T>(text: string, languageVariant: LanguageV
}
function skipToStartOf(node: Node): void {
scanner.setTextPos(node.pos);
savedPos = scanner.getStartPos();
scanner.resetTokenState(node.pos);
savedPos = scanner.getTokenFullStart();
lastScanAction = undefined;
lastTokenInfo = undefined;
wasNewLine = false;
src/services/organizeImports.ts
+1 -1
View File
@@ -202,7 +202,7 @@ function isNewGroup(sourceFile: SourceFile, decl: ImportDeclaration | ExportDecl
scanner.setText(sourceFile.text, startPos, endPos - startPos);
let numberOfNewLines = 0;
while (scanner.getTokenPos() < endPos) {
while (scanner.getTokenStart() < endPos) {
const tokenKind = scanner.scan();
if (tokenKind === SyntaxKind.NewLineTrivia) {
src/services/preProcess.ts
+1 -1
View File
@@ -48,7 +48,7 @@ export function preProcessFile(sourceText: string, readImportFiles = true, detec
function getFileReference() {
const fileName = scanner.getTokenValue();
const pos = scanner.getTokenPos();
const pos = scanner.getTokenStart();
return { fileName, pos, end: pos + fileName.length };
}
src/services/services.ts
+2 -2
View File
@@ -487,10 +487,10 @@ function createChildren(node: Node, sourceFile: SourceFileLike | undefined): Nod
}
function addSyntheticNodes(nodes: Node[], pos: number, end: number, parent: Node): void {
scanner.setTextPos(pos);
scanner.resetTokenState(pos);
while (pos < end) {
const token = scanner.scan();
const textPos = scanner.getTextPos();
const textPos = scanner.getTokenEnd();
if (textPos <= end) {
if (token === SyntaxKind.Identifier) {
if (hasTabstop(parent)) {
tests/baselines/reference/api/tsserverlibrary.d.ts
Vendored
+8
View File
@@ -8363,9 +8363,15 @@ declare namespace ts {
function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, languageVariant?: LanguageVariant, textInitial?: string, onError?: ErrorCallback, start?: number, length?: number): Scanner;
type ErrorCallback = (message: DiagnosticMessage, length: number) => void;
interface Scanner {
/** @deprecated use {@link getTokenFullStart} */
getStartPos(): number;
getToken(): SyntaxKind;
getTokenFullStart(): number;
getTokenStart(): number;
getTokenEnd(): number;
/** @deprecated use {@link getTokenEnd} */
getTextPos(): number;
/** @deprecated use {@link getTokenStart} */
getTokenPos(): number;
getTokenText(): string;
getTokenValue(): string;
@@ -8396,7 +8402,9 @@ declare namespace ts {
setOnError(onError: ErrorCallback | undefined): void;
setScriptTarget(scriptTarget: ScriptTarget): void;
setLanguageVariant(variant: LanguageVariant): void;
/** @deprecated use {@link resetTokenState} */
setTextPos(textPos: number): void;
resetTokenState(pos: number): void;
lookAhead<T>(callback: () => T): T;
scanRange<T>(start: number, length: number, callback: () => T): T;
tryScan<T>(callback: () => T): T;
tests/baselines/reference/api/typescript.d.ts
Vendored
+8
View File
@@ -4420,9 +4420,15 @@ declare namespace ts {
function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, languageVariant?: LanguageVariant, textInitial?: string, onError?: ErrorCallback, start?: number, length?: number): Scanner;
type ErrorCallback = (message: DiagnosticMessage, length: number) => void;
interface Scanner {
/** @deprecated use {@link getTokenFullStart} */
getStartPos(): number;
getToken(): SyntaxKind;
getTokenFullStart(): number;
getTokenStart(): number;
getTokenEnd(): number;
/** @deprecated use {@link getTokenEnd} */
getTextPos(): number;
/** @deprecated use {@link getTokenStart} */
getTokenPos(): number;
getTokenText(): string;
getTokenValue(): string;
@@ -4453,7 +4459,9 @@ declare namespace ts {
setOnError(onError: ErrorCallback | undefined): void;
setScriptTarget(scriptTarget: ScriptTarget): void;
setLanguageVariant(variant: LanguageVariant): void;
/** @deprecated use {@link resetTokenState} */
setTextPos(textPos: number): void;
resetTokenState(pos: number): void;
lookAhead<T>(callback: () => T): T;
scanRange<T>(start: number, length: number, callback: () => T): T;
tryScan<T>(callback: () => T): T;