Microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2025-11-18 17:21:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Set startPos at EOF in jsdoc token scanner so node end positions for nodes terminated at EoF are right (#24184)

Browse Source 
* Set startPos at EOF in jsdoc token scanner to node end positions for nodes terminated at EoF are right

* More complete nonwhitespace token check, fix syntactica jsdoc classifier

* Use loop and no nested lookahead

* Do thigns unrelated to the bug in the test

* Fix typo move return

* Patch up typedef end pos

* Fix indentation, make end pos target more obvious
This commit is contained in:
Wesley Wigham
2018-05-17 15:16:18 -07:00
committed by GitHub
parent d579793d0a
commit d82d35c7f5
23 changed files with 112 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/compiler/parser.ts
+23 -2
View File
@@ -6481,7 +6481,25 @@ namespace ts {
return finishNode(result, end);
}
function isNextNonwhitespaceTokenEndOfFile(): boolean {
// We must use infinite lookahead, as there could be any number of newlines :(
while (true) {
nextJSDocToken();
if (token() === SyntaxKind.EndOfFileToken) {
return true;
}
if (!(token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia)) {
return false;
}
}
}
function skipWhitespace(): void {
if (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) {
if (lookAhead(isNextNonwhitespaceTokenEndOfFile)) {
return; // Don't skip whitespace prior to EoF (or end of comment) - that shouldn't be included in any node's range
}
}
while (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) {
nextJSDocToken();
}
@@ -6802,6 +6820,7 @@ namespace ts {
typedefTag.comment = parseTagComments(indent);
typedefTag.typeExpression = typeExpression;
let end: number;
if (!typeExpression || isObjectOrObjectArrayTypeReference(typeExpression.type)) {
let child: JSDocTypeTag | JSDocPropertyTag | false;
let jsdocTypeLiteral: JSDocTypeLiteral;
@@ -6830,10 +6849,12 @@ namespace ts {
typedefTag.typeExpression = childTypeTag && childTypeTag.typeExpression && !isObjectOrObjectArrayTypeReference(childTypeTag.typeExpression.type) ?
childTypeTag.typeExpression :
finishNode(jsdocTypeLiteral);
end = typedefTag.typeExpression.end;
}
}
return finishNode(typedefTag);
// Only include the characters between the name end and the next token if a comment was actually parsed out - otherwise it's just whitespace
return finishNode(typedefTag, end || typedefTag.comment !== undefined ? scanner.getStartPos() : (typedefTag.fullName || typedefTag.typeExpression || typedefTag.tagName).end);
}
function parseJSDocTypeNameWithNamespace(nested?: boolean) {
@@ -7075,7 +7096,7 @@ namespace ts {
const pos = scanner.getTokenPos();
const end = scanner.getTextPos();
const result = <Identifier>createNode(SyntaxKind.Identifier, pos);
result.escapedText = escapeLeadingUnderscores(content.substring(pos, end));
result.escapedText = escapeLeadingUnderscores(scanner.getTokenText());
finishNode(result, end);
nextJSDocToken();
src/compiler/scanner.ts
+1 -3
View File
@@ -1928,13 +1928,11 @@ namespace ts {
}
function scanJSDocToken(): JsDocSyntaxKind {
startPos = tokenPos = pos;
if (pos >= end) {
return token = SyntaxKind.EndOfFileToken;
}
startPos = pos;
tokenPos = pos;
const ch = text.charCodeAt(pos);
pos++;
switch (ch) {
src/services/classifier.ts
+3 -2
View File
@@ -706,16 +706,17 @@ namespace ts {
break;
case SyntaxKind.JSDocTemplateTag:
processJSDocTemplateTag(<JSDocTemplateTag>tag);
pos = tag.end;
break;
case SyntaxKind.JSDocTypeTag:
processElement((<JSDocTypeTag>tag).typeExpression);
pos = tag.end;
break;
case SyntaxKind.JSDocReturnTag:
processElement((<JSDocReturnTag>tag).typeExpression);
pos = tag.end;
break;
}
pos = tag.end;
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.Nested @param tags.json
+4 -4
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 6,
"end": 63,
"end": 64,
"atToken": {
"kind": "AtToken",
"pos": 6,
@@ -21,11 +21,11 @@
"typeExpression": {
"kind": "JSDocTypeExpression",
"pos": 34,
"end": 63,
"end": 64,
"type": {
"kind": "JSDocTypeLiteral",
"pos": 34,
"end": 63,
"end": 64,
"jsDocPropertyTags": [
{
"kind": "JSDocParameterTag",
@@ -88,6 +88,6 @@
},
"length": 1,
"pos": 6,
"end": 63
"end": 64
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argSynonymForParamTag.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 8,
"end": 40,
"end": 42,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -40,6 +40,6 @@
},
"length": 1,
"pos": 8,
"end": 40
"end": 42
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argumentSynonymForParamTag.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 8,
"end": 45,
"end": 47,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -40,6 +40,6 @@
},
"length": 1,
"pos": 8,
"end": 45
"end": 47
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.less-than and greater-than characters.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 7,
"end": 58,
"end": 59,
"atToken": {
"kind": "AtToken",
"pos": 7,
@@ -30,6 +30,6 @@
},
"length": 1,
"pos": 7,
"end": 58
"end": 59
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noReturnType.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocReturnTag",
"pos": 8,
"end": 16,
"end": 15,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -21,6 +21,6 @@
},
"length": 1,
"pos": 8,
"end": 16
"end": 15
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.oneParamTag.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 8,
"end": 30,
"end": 32,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -39,6 +39,6 @@
},
"length": 1,
"pos": 8,
"end": 30
"end": 32
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTag1.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 8,
"end": 55,
"end": 57,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -40,6 +40,6 @@
},
"length": 1,
"pos": 8,
"end": 55
"end": 57
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName1.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 8,
"end": 57,
"end": 59,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -40,6 +40,6 @@
},
"length": 1,
"pos": 8,
"end": 57
"end": 59
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName2.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 8,
"end": 62,
"end": 64,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -40,6 +40,6 @@
},
"length": 1,
"pos": 8,
"end": 62
"end": 64
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagNameThenType2.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 8,
"end": 42,
"end": 44,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -40,6 +40,6 @@
},
"length": 1,
"pos": 8,
"end": 42
"end": 44
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramWithoutType.json
+2 -2
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocParameterTag",
"pos": 8,
"end": 19,
"end": 21,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -29,6 +29,6 @@
},
"length": 1,
"pos": 8,
"end": 19
"end": 21
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag.json
+4 -4
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocTemplateTag",
"pos": 8,
"end": 20,
"end": 19,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -22,7 +22,7 @@
"0": {
"kind": "TypeParameter",
"pos": 18,
"end": 20,
"end": 19,
"name": {
"kind": "Identifier",
"pos": 18,
@@ -32,11 +32,11 @@
},
"length": 1,
"pos": 18,
"end": 20
"end": 19
}
},
"length": 1,
"pos": 8,
"end": 20
"end": 19
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag2.json
+4 -4
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocTemplateTag",
"pos": 8,
"end": 22,
"end": 21,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -33,7 +33,7 @@
"1": {
"kind": "TypeParameter",
"pos": 20,
"end": 22,
"end": 21,
"name": {
"kind": "Identifier",
"pos": 20,
@@ -43,11 +43,11 @@
},
"length": 2,
"pos": 18,
"end": 22
"end": 21
}
},
"length": 1,
"pos": 8,
"end": 22
"end": 21
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag3.json
+4 -4
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocTemplateTag",
"pos": 8,
"end": 23,
"end": 22,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -33,7 +33,7 @@
"1": {
"kind": "TypeParameter",
"pos": 21,
"end": 23,
"end": 22,
"name": {
"kind": "Identifier",
"pos": 21,
@@ -43,11 +43,11 @@
},
"length": 2,
"pos": 18,
"end": 23
"end": 22
}
},
"length": 1,
"pos": 8,
"end": 23
"end": 22
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag4.json
+4 -4
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocTemplateTag",
"pos": 8,
"end": 23,
"end": 22,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -33,7 +33,7 @@
"1": {
"kind": "TypeParameter",
"pos": 21,
"end": 23,
"end": 22,
"name": {
"kind": "Identifier",
"pos": 21,
@@ -43,11 +43,11 @@
},
"length": 2,
"pos": 18,
"end": 23
"end": 22
}
},
"length": 1,
"pos": 8,
"end": 23
"end": 22
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag5.json
+4 -4
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocTemplateTag",
"pos": 8,
"end": 24,
"end": 23,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -33,7 +33,7 @@
"1": {
"kind": "TypeParameter",
"pos": 22,
"end": 24,
"end": 23,
"name": {
"kind": "Identifier",
"pos": 22,
@@ -43,11 +43,11 @@
},
"length": 2,
"pos": 18,
"end": 24
"end": 23
}
},
"length": 1,
"pos": 8,
"end": 24
"end": 23
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTag2.json
+2 -2
View File
@@ -40,7 +40,7 @@
"1": {
"kind": "JSDocParameterTag",
"pos": 34,
"end": 56,
"end": 58,
"atToken": {
"kind": "AtToken",
"pos": 34,
@@ -73,6 +73,6 @@
},
"length": 2,
"pos": 8,
"end": 56
"end": 58
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTagOnSameLine.json
+2 -2
View File
@@ -40,7 +40,7 @@
"1": {
"kind": "JSDocParameterTag",
"pos": 30,
"end": 52,
"end": 54,
"atToken": {
"kind": "AtToken",
"pos": 30,
@@ -73,6 +73,6 @@
},
"length": 2,
"pos": 8,
"end": 52
"end": 54
}
}
tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.typedefTagWithChildrenTags.json
+4 -4
View File
@@ -6,7 +6,7 @@
"0": {
"kind": "JSDocTypedefTag",
"pos": 8,
"end": 98,
"end": 100,
"atToken": {
"kind": "AtToken",
"pos": 8,
@@ -33,7 +33,7 @@
"typeExpression": {
"kind": "JSDocTypeLiteral",
"pos": 28,
"end": 98,
"end": 100,
"jsDocPropertyTags": [
{
"kind": "JSDocPropertyTag",
@@ -72,7 +72,7 @@
{
"kind": "JSDocPropertyTag",
"pos": 74,
"end": 98,
"end": 97,
"atToken": {
"kind": "AtToken",
"pos": 74,
@@ -108,6 +108,6 @@
},
"length": 1,
"pos": 8,
"end": 98
"end": 100
}
}
tests/cases/fourslash/incrementalJsDocAdjustsLengthsRight.ts
+33
View File
@@ -0,0 +1,33 @@
/// <reference path="fourslash.ts" />
// @noLib: true
////
/////**
//// * Pad `str` to `width`.
//// *
//// * @param {String} str
//// * @param {Number} wid/*1*/
goTo.marker('1');
edit.insert("th\n@");
const c = classification;
verify.syntacticClassificationsAre(
c.comment("/**\n * Pad `str` to `width`.\n *\n * "),
c.punctuation("@"),
c.docCommentTagName("param"),
c.comment(" "),
c.punctuation("{"),
c.identifier("String"),
c.punctuation("}"),
c.comment(" "),
c.parameterName("str"),
c.comment("\n * "),
c.punctuation("@"),
c.docCommentTagName("param"),
c.comment(" "),
c.punctuation("{"),
c.identifier("Number"),
c.punctuation("}"),
c.comment(" "),
c.parameterName("wid"),
c.comment(""), // syntatic classification verification always just uses input text, so the edits don't appear
);