Microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2025-11-18 17:21:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Parse json using our own parser

Browse Source
This commit is contained in:
Sheetal Nandi
2016-11-17 14:45:01 -08:00
parent c90a40c58f
commit cca98c308f
12 changed files with 172 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/compiler/commandLineParser.ts
+99 -35
View File
@@ -681,13 +681,13 @@ namespace ts {
* Read tsconfig.json file
* @param fileName The path to the config file
*/
export function readConfigFile(fileName: string, readFile: (path: string) => string): { config?: any; error?: Diagnostic } {
export function readConfigFile(fileName: string, readFile: (path: string) => string): { config?: any; errors: Diagnostic[] } {
let text = "";
try {
text = readFile(fileName);
}
catch (e) {
return { error: createCompilerDiagnostic(Diagnostics.Cannot_read_file_0_Colon_1, fileName, e.message) };
return { errors: [createCompilerDiagnostic(Diagnostics.Cannot_read_file_0_Colon_1, fileName, e.message)] };
}
return parseConfigFileTextToJson(fileName, text);
}
@@ -697,13 +697,102 @@ namespace ts {
* @param fileName The path to the config file
* @param jsonText The text of the config file
*/
export function parseConfigFileTextToJson(fileName: string, jsonText: string, stripComments = true): { config?: any; error?: Diagnostic } {
try {
const jsonTextToParse = stripComments ? removeComments(jsonText) : jsonText;
return { config: /\S/.test(jsonTextToParse) ? JSON.parse(jsonTextToParse) : {} };
export function parseConfigFileTextToJson(fileName: string, jsonText: string): { config: any; errors: Diagnostic[] } {
const { node, errors } = parseJsonText(fileName, jsonText);
return {
config: convertToJson(node, errors),
errors
};
}
/**
* Convert the json syntax tree into the json value
* @param jsonNode
* @param errors
*/
function convertToJson(jsonNode: JsonNode, errors: Diagnostic[]): any {
if (!jsonNode) {
return undefined;
}
catch (e) {
return { error: createCompilerDiagnostic(Diagnostics.Failed_to_parse_file_0_Colon_1, fileName, e.message) };
if (jsonNode.kind === SyntaxKind.EndOfFileToken) {
return {};
}
const sourceFile = <SourceFile>jsonNode.parent;
return convertObjectLiteralExpressionToJson(jsonNode);
function convertObjectLiteralExpressionToJson(node: ObjectLiteralExpression): any {
const result: any = {};
for (const element of node.properties) {
switch (element.kind) {
case SyntaxKind.MethodDeclaration:
case SyntaxKind.GetAccessor:
case SyntaxKind.SetAccessor:
case SyntaxKind.ShorthandPropertyAssignment:
case SyntaxKind.SpreadAssignment:
errors.push(createDiagnosticForNodeInSourceFile(sourceFile, element, Diagnostics.Property_assignment_expected));
break;
case SyntaxKind.PropertyAssignment:
if (element.questionToken) {
errors.push(createDiagnosticForNodeInSourceFile(sourceFile, element.questionToken, Diagnostics._0_can_only_be_used_in_a_ts_file, "?"));
}
if (!isDoubleQuotedString(element.name)) {
errors.push(createDiagnosticForNodeInSourceFile(sourceFile, element.name, Diagnostics.String_literal_with_double_quotes_expected));
}
const keyText = getTextOfPropertyName(element.name);
const value = parseValue(element.initializer);
if (typeof keyText !== undefined && typeof value !== undefined) {
result[keyText] = value;
}
}
}
return result;
}
function convertArrayLiteralExpressionToJson(node: ArrayLiteralExpression): any[] {
const result: any[] = [];
for (const element of node.elements) {
result.push(parseValue(element));
}
return result;
}
function parseValue(node: Expression): any {
switch (node.kind) {
case SyntaxKind.TrueKeyword:
return true;
case SyntaxKind.FalseKeyword:
return false;
case SyntaxKind.NullKeyword:
return null; // tslint:disable-line:no-null-keyword
case SyntaxKind.StringLiteral:
if (!isDoubleQuotedString(node)) {
errors.push(createDiagnosticForNodeInSourceFile(sourceFile, node, Diagnostics.String_literal_with_double_quotes_expected));
}
return (<StringLiteral>node).text;
case SyntaxKind.NumericLiteral:
return Number((<NumericLiteral>node).text);
case SyntaxKind.ObjectLiteralExpression:
return convertObjectLiteralExpressionToJson(<ObjectLiteralExpression>node);
case SyntaxKind.ArrayLiteralExpression:
return convertArrayLiteralExpressionToJson(<ArrayLiteralExpression>node);
}
// Not in expected format
errors.push(createDiagnosticForNodeInSourceFile(sourceFile, node, Diagnostics.String_number_object_array_true_false_or_null_expected));
return undefined;
}
function isDoubleQuotedString(node: Node) {
return node.kind === SyntaxKind.StringLiteral && getSourceTextOfNodeFromSourceFile(sourceFile, node).charCodeAt(0) === CharacterCodes.doubleQuote;
}
}
@@ -795,31 +884,6 @@ namespace ts {
}
}
/**
* Remove the comments from a json like text.
* Comments can be single line comments (starting with # or //) or multiline comments using / * * /
*
* This method replace comment content by whitespace rather than completely remove them to keep positions in json parsing error reporting accurate.
*/
function removeComments(jsonText: string): string {
let output = "";
const scanner = createScanner(ScriptTarget.ES5, /* skipTrivia */ false, LanguageVariant.Standard, jsonText);
let token: SyntaxKind;
while ((token = scanner.scan()) !== SyntaxKind.EndOfFileToken) {
switch (token) {
case SyntaxKind.SingleLineCommentTrivia:
case SyntaxKind.MultiLineCommentTrivia:
// replace comments with whitespace to preserve original character positions
output += scanner.getTokenText().replace(/\S/g, " ");
break;
default:
output += scanner.getTokenText();
break;
}
}
return output;
}
/**
* Parse the contents of a config file (tsconfig.json).
* @param json The contents of the config file to parse
@@ -896,8 +960,8 @@ namespace ts {
}
}
const extendedResult = readConfigFile(extendedConfigPath, path => host.readFile(path));
if (extendedResult.error) {
errors.push(extendedResult.error);
if (extendedResult.errors.length) {
errors.push(...extendedResult.errors);
return;
}
const extendedDirname = getDirectoryPath(extendedConfigPath);
src/compiler/diagnosticMessages.json
+8
View File
@@ -851,6 +851,14 @@
"category": "Error",
"code": 1317
},
"String literal with double quotes expected.": {
"category": "Error",
"code": 1318
},
"String, number, object, array, true, false or null expected.": {
"category": "Error",
"code": 1319
},
"Duplicate identifier '{0}'.": {
"category": "Error",
"code": 2300
src/compiler/parser.ts
+38
View File
@@ -449,6 +449,17 @@ namespace ts {
return Parser.parseIsolatedEntityName(text, languageVersion);
}
export type ParsedNodeResults<T extends Node> = { node: T; errors: Diagnostic[] };
/**
* Parse json text into SyntaxTree and return node and parse errors if any
* @param fileName
* @param sourceText
*/
export function parseJsonText(fileName: string, sourceText: string): ParsedNodeResults<JsonNode> {
return Parser.parseJsonText(fileName, sourceText);
}
export function isExternalModule(file: SourceFile): boolean {
return file.externalModuleIndicator !== undefined;
}
@@ -610,6 +621,33 @@ namespace ts {
return isInvalid ? entityName : undefined;
}
export function parseJsonText(fileName: string, sourceText: string): ParsedNodeResults<JsonNode> {
initializeState(sourceText, ScriptTarget.ES2015, /*syntaxCursor*/ undefined, ScriptKind.JS);
// Set source file so that errors will be reported with this file name
sourceFile = <SourceFile>{ kind: SyntaxKind.SourceFile, text: sourceText, fileName };
let node: JsonNode;
// Prime the scanner.
nextToken();
if (token() === SyntaxKind.EndOfFileToken) {
node = <EndOfFileToken>parseTokenNode();
}
else if (token() === SyntaxKind.OpenBraceToken ||
lookAhead(() => token() === SyntaxKind.StringLiteral)) {
node = parseObjectLiteralExpression();
parseExpected(SyntaxKind.EndOfFileToken, Diagnostics.Unexpected_token);
}
else {
parseExpected(SyntaxKind.OpenBraceToken);
}
if (node) {
node.parent = sourceFile;
}
const errors = parseDiagnostics;
clearState();
return { node, errors };
}
function getLanguageVariant(scriptKind: ScriptKind) {
// .tsx and .jsx files are treated as jsx language variant.
return scriptKind === ScriptKind.TSX || scriptKind === ScriptKind.JSX || scriptKind === ScriptKind.JS ? LanguageVariant.JSX : LanguageVariant.Standard;
src/compiler/tsc.ts
+1 -1
View File
@@ -367,9 +367,9 @@ namespace ts {
}
const result = parseConfigFileTextToJson(configFileName, cachedConfigFileText);
reportDiagnostics(result.errors, /* compilerHost */ undefined);
const configObject = result.config;
if (!configObject) {
reportDiagnostics([result.error], /* compilerHost */ undefined);
sys.exit(ExitStatus.DiagnosticsPresent_OutputsSkipped);
return;
}
src/compiler/types.ts
+2
View File
@@ -525,6 +525,8 @@ namespace ts {
export type AtToken = Token<SyntaxKind.AtToken>;
export type ReadonlyToken = Token<SyntaxKind.ReadonlyKeyword>;
export type JsonNode = ObjectLiteralExpression | EndOfFileToken;
export type Modifier
= Token<SyntaxKind.AbstractKeyword>
| Token<SyntaxKind.AsyncKeyword>
src/harness/projectsRunner.ts
+6 -4
View File
@@ -208,12 +208,13 @@ class ProjectRunner extends RunnerBase {
configFileName = ts.findConfigFile("", fileExists);
}
let errors: ts.Diagnostic[];
if (configFileName) {
const result = ts.readConfigFile(configFileName, getSourceFileText);
if (result.error) {
if (!result.config) {
return {
moduleKind,
errors: [result.error]
errors: result.errors
};
}
@@ -228,11 +229,12 @@ class ProjectRunner extends RunnerBase {
if (configParseResult.errors.length > 0) {
return {
moduleKind,
errors: configParseResult.errors
errors: result.errors.concat(configParseResult.errors)
};
}
inputFiles = configParseResult.fileNames;
compilerOptions = configParseResult.options;
errors = result.errors;
}
const projectCompilerResult = compileProjectFiles(moduleKind, () => inputFiles, getSourceFileText, writeFile, compilerOptions);
@@ -242,7 +244,7 @@ class ProjectRunner extends RunnerBase {
compilerOptions,
sourceMapData: projectCompilerResult.sourceMapData,
outputFiles,
errors: projectCompilerResult.errors,
errors: errors ? errors.concat(projectCompilerResult.errors) : projectCompilerResult.errors,
};
function createCompilerOptions() {
src/harness/unittests/configurationExtension.ts
+4 -4
View File
@@ -112,8 +112,8 @@ namespace ts {
], ([testName, basePath, host]) => {
function testSuccess(name: string, entry: string, expected: CompilerOptions, expectedFiles: string[]) {
it(name, () => {
const {config, error} = ts.readConfigFile(entry, name => host.readFile(name));
assert(config && !error, flattenDiagnosticMessageText(error && error.messageText, "\n"));
const {config, errors} = ts.readConfigFile(entry, name => host.readFile(name));
assert(config && !errors.length, flattenDiagnosticMessageText(errors[0] && errors[0].messageText, "\n"));
const parsed = ts.parseJsonConfigFileContent(config, host, basePath, {}, entry);
assert(!parsed.errors.length, flattenDiagnosticMessageText(parsed.errors[0] && parsed.errors[0].messageText, "\n"));
expected.configFilePath = entry;
@@ -124,8 +124,8 @@ namespace ts {
function testFailure(name: string, entry: string, expectedDiagnostics: {code: number, category: DiagnosticCategory, messageText: string}[]) {
it(name, () => {
const {config, error} = ts.readConfigFile(entry, name => host.readFile(name));
assert(config && !error, flattenDiagnosticMessageText(error && error.messageText, "\n"));
const {config, errors} = ts.readConfigFile(entry, name => host.readFile(name));
assert(config && !errors.length, flattenDiagnosticMessageText(errors[0] && errors[0].messageText, "\n"));
const parsed = ts.parseJsonConfigFileContent(config, host, basePath, {}, entry);
verifyDiagnostics(parsed.errors, expectedDiagnostics);
});
src/harness/unittests/projectErrors.ts
+2 -8
View File
@@ -123,10 +123,7 @@ namespace ts.projectSystem {
const configuredProject = forEach(projectService.synchronizeProjectList([]), f => f.info.projectName === corruptedConfig.path && f);
assert.isTrue(configuredProject !== undefined, "should find configured project");
checkProjectErrors(configuredProject, [
"')' expected.",
"Declaration or statement expected.",
"Declaration or statement expected.",
"Failed to parse file '/a/b/tsconfig.json'"
"'{' expected."
]);
}
// fix config and trigger watcher
@@ -175,10 +172,7 @@ namespace ts.projectSystem {
const configuredProject = forEach(projectService.synchronizeProjectList([]), f => f.info.projectName === corruptedConfig.path && f);
assert.isTrue(configuredProject !== undefined, "should find configured project");
checkProjectErrors(configuredProject, [
"')' expected.",
"Declaration or statement expected.",
"Declaration or statement expected.",
"Failed to parse file '/a/b/tsconfig.json'"
"'{' expected."
]);
}
});
src/harness/unittests/tsconfigParsing.ts
+6 -3
View File
@@ -3,15 +3,18 @@
namespace ts {
describe("parseConfigFileTextToJson", () => {
function assertParseResult(jsonText: string, expectedConfigObject: { config?: any; error?: Diagnostic }) {
function assertParseResult(jsonText: string, expectedConfigObject: { config?: any; errors?: Diagnostic[] }) {
const parsed = ts.parseConfigFileTextToJson("/apath/tsconfig.json", jsonText);
if (!expectedConfigObject.errors) {
expectedConfigObject.errors = [];
}
assert.equal(JSON.stringify(parsed), JSON.stringify(expectedConfigObject));
}
function assertParseError(jsonText: string) {
const parsed = ts.parseConfigFileTextToJson("/apath/tsconfig.json", jsonText);
assert.isTrue(undefined === parsed.config);
assert.isTrue(undefined !== parsed.error);
assert.isTrue(!!parsed.errors.length);
}
function assertParseErrorWithExcludesKeyword(jsonText: string) {
@@ -199,7 +202,7 @@ namespace ts {
}
"files": ["file1.ts"]
}`;
const { configJsonObject, diagnostics } = sanitizeConfigFile("config.json", content);
const { config: configJsonObject, errors: diagnostics } = parseConfigFileTextToJson("config.json", content);
const expectedResult = {
compilerOptions: {
allowJs: true,
src/server/editorServices.ts
+3 -13
View File
@@ -786,18 +786,8 @@ namespace ts.server {
configFilename = normalizePath(configFilename);
const configFileContent = this.host.readFile(configFilename);
let errors: Diagnostic[];
const result = parseConfigFileTextToJson(configFilename, configFileContent);
let config = result.config;
if (result.error) {
// try to reparse config file
const { configJsonObject: sanitizedConfig, diagnostics } = sanitizeConfigFile(configFilename, configFileContent);
config = sanitizedConfig;
errors = diagnostics.length ? diagnostics : [result.error];
}
const { config = {}, errors } = parseConfigFileTextToJson(configFilename, configFileContent);
const parsedCommandLine = parseJsonConfigFileContent(
config,
this.host,
@@ -806,13 +796,13 @@ namespace ts.server {
configFilename);
if (parsedCommandLine.errors.length) {
errors = concatenate(errors, parsedCommandLine.errors);
errors.push(...parsedCommandLine.errors);
}
Debug.assert(!!parsedCommandLine.fileNames);
if (parsedCommandLine.fileNames.length === 0) {
(errors || (errors = [])).push(createCompilerDiagnostic(Diagnostics.The_config_file_0_found_doesn_t_contain_any_source_files, configFilename));
errors.push(createCompilerDiagnostic(Diagnostics.The_config_file_0_found_doesn_t_contain_any_source_files, configFilename));
return { success: false, configFileErrors: errors };
}
src/services/shims.ts
+3 -3
View File
@@ -1135,13 +1135,13 @@ namespace ts {
const result = parseConfigFileTextToJson(fileName, text);
if (result.error) {
if (!result.config) {
return {
options: {},
typingOptions: {},
files: [],
raw: {},
errors: [realizeDiagnostic(result.error, "\r\n")]
errors: realizeDiagnostics(result.errors, "\r\n")
};
}
@@ -1153,7 +1153,7 @@ namespace ts {
typingOptions: configFile.typingOptions,
files: configFile.fileNames,
raw: configFile.raw,
errors: realizeDiagnostics(configFile.errors, "\r\n")
errors: realizeDiagnostics(result.errors.concat(configFile.errors), "\r\n")
};
});
}
src/services/utilities.ts
-24
View File
@@ -1334,28 +1334,4 @@ namespace ts {
}
return ensureScriptKind(fileName, scriptKind);
}
export function sanitizeConfigFile(configFileName: string, content: string) {
const options: TranspileOptions = {
fileName: "config.js",
compilerOptions: {
target: ScriptTarget.ES2015,
removeComments: true
},
reportDiagnostics: true
};
const { outputText, diagnostics } = ts.transpileModule("(" + content + ")", options);
// Becasue the content was wrapped in "()", the start position of diagnostics needs to be subtract by 1
// also, the emitted result will have "(" in the beginning and ");" in the end. We need to strip these
// as well
const trimmedOutput = outputText.trim();
for (const diagnostic of diagnostics) {
diagnostic.start = diagnostic.start - 1;
}
const {config, error} = parseConfigFileTextToJson(configFileName, trimmedOutput.substring(1, trimmedOutput.length - 2), /*stripComments*/ false);
return {
configJsonObject: config || {},
diagnostics: error ? concatenate(diagnostics, [error]) : diagnostics
};
}
}