[rust] Partial ES2015 support in estree codegen

Adds some parts of the ES2015 spec, such as imports and ForOfStatement. This is 
enough to get a few more fixtures compiling. The last one uses JSX which I 
haven't defined yet.

compiler/forget/crates/estree-codegen/src/codegen.rs

@@ -168,6 +168,9 @@ pub struct Field {
 
     #[serde(default)]
     pub flatten: bool,
+
+    #[serde(default)]
+    pub rename: Option<String>,
 }
 
 impl Field {
@@ -193,11 +196,7 @@ impl Field {
                 "Can only set nullable_item if plural"
             )
         }
-        if self.nullable || self.optional {
-            assert_eq!(
-                self.optional, false,
-                "Expected field to be nullable if optional"
-            );
+        if self.nullable {
             type_ = quote!(Option<#type_>);
         }
         let mut field = quote!(#name: #type_);
@@ -207,6 +206,12 @@ impl Field {
                 #field
             }
         }
+        if let Some(rename) = &self.rename {
+            field = quote! {
+                #[serde(rename = #rename)]
+                #field
+            }
+        }
         if self.flatten {
             field = quote! {
                 #[serde(flatten)]
@@ -238,11 +243,7 @@ impl Field {
                 "Can only set nullable_item if plural"
             )
         }
-        if self.nullable || self.optional {
-            assert_eq!(
-                self.nullable, true,
-                "Expected field to be nullable if optional"
-            );
+        if self.nullable {
             type_ = quote!(Option<#type_>);
         }
         let mut field = quote!(#name: #type_);
@@ -258,6 +259,12 @@ impl Field {
                 #field
             }
         }
+        if let Some(rename) = &self.rename {
+            field = quote! {
+                #[serde(rename = #rename)]
+                #field
+            }
+        }
         if nodes.contains(&self.type_) {
             let comment = format!(" {}", &self.type_);
             field = quote! {

compiler/forget/crates/estree-codegen/src/ecmascript.json

@@ -35,8 +35,18 @@
                     "plural": true
                 },
                 "body": {
-                    "type": "BlockStatement",
+                    "type": "FunctionBody",
                     "nullable": true
+                },
+                "is_generator": {
+                    "type": "bool",
+                    "optional": true,
+                    "rename": "generator"
+                },
+                "is_async": {
+                    "type": "bool",
+                    "optional": true,
+                    "rename": "async"
                 }
             }
         },
@@ -85,8 +95,13 @@
         "Program": {
             "fields": {
                 "body": {
-                    "type": "Statement",
+                    "type": "ModuleItem",
                     "plural": true
+                },
+                "source_type": {
+                    "type": "SourceType",
+                    "nullable": true,
+                    "rename": "sourceType"
                 }
             }
         },
@@ -94,6 +109,11 @@
             "fields": {
                 "expression": {
                     "type": "Expression"
+                },
+                "directive": {
+                    "type": "String",
+                    "nullable": true,
+                    "optional": true
                 }
             }
         },
@@ -272,6 +292,19 @@
                 }
             }
         },
+        "ForOfStatement": {
+            "fields": {
+                "left": {
+                    "type": "ForInInit"
+                },
+                "right": {
+                    "type": "Expression"
+                },
+                "body": {
+                    "type": "Statement"
+                }
+            }
+        },
         "FunctionDeclaration": {
             "fields": {
                 "function": {
@@ -306,7 +339,7 @@
         "ArrayExpression": {
             "fields": {
                 "elements": {
-                    "type": "Expression",
+                    "type": "ExpressionOrSpread",
                     "plural": true,
                     "nullable_item": true
                 }
@@ -341,6 +374,18 @@
                 }
             }
         },
+        "ArrowFunctionExpression": {
+            "fields": {
+                "function": {
+                    "type": "Function",
+                    "flatten": true
+                },
+                "is_expression": {
+                    "type": "bool",
+                    "rename": "expression"
+                }
+            }
+        },
         "UnaryExpression": {
             "fields": {
                 "operator": {
@@ -409,7 +454,7 @@
         "MemberExpression": {
             "fields": {
                 "object": {
-                    "type": "Expression"
+                    "type": "ExpressionOrSuper"
                 },
                 "property": {
                     "type": "Expression"
@@ -433,37 +478,90 @@
             }
         },
         "CallExpression": {
-            "type": true,
             "fields": {
                 "callee": {
-                    "type": "Expression"
+                    "type": "ExpressionOrSuper"
                 },
                 "arguments": {
-                    "type": "Expression",
+                    "type": "ExpressionOrSpread",
                     "plural": true
                 }
             }
         },
         "NewExpression": {
-            "type": true,
             "fields": {
                 "callee": {
                     "type": "Expression"
                 },
                 "arguments": {
-                    "type": "Expression",
+                    "type": "ExpressionOrSpread",
                     "plural": true
                 }
             }
         },
         "SequenceExpression": {
-            "type": true,
             "fields": {
                 "expressions": {
                     "type": "Expression",
                     "plural": true
                 }
             }
+        },
+        "Super": {},
+        "SpreadElement": {
+            "fields": {
+                "argument": {
+                    "type": "Expression"
+                }
+            }
+        },
+        "YieldExpression": {
+            "fields": {
+                "argument": {
+                    "type": "Expression",
+                    "nullable": true,
+                    "optional": true
+                },
+                "is_delegate": {
+                    "type": "bool",
+                    "rename": "delegate"
+                }
+            }
+        },
+        "ImportDeclaration": {
+            "fields": {
+                "specifiers": {
+                    "type": "ImportDeclarationSpecifier",
+                    "plural": true
+                },
+                "source": {
+                    "type": "Literal"
+                }
+            }
+        },
+        "ImportSpecifier": {
+            "fields": {
+                "imported": {
+                    "type": "Identifier"
+                },
+                "local": {
+                    "type": "Identifier"
+                }
+            }
+        },
+        "ImportDefaultSpecifier": {
+            "fields": {
+                "local": {
+                    "type": "Identifier"
+                }
+            }
+        },
+        "ImportNamespaceSpecifier": {
+            "fields": {
+                "local": {
+                    "type": "Identifier"
+                }
+            }
         }
     },
     "enums": {
@@ -476,6 +574,7 @@
             "EmptyStatement",
             "ExpressionStatement",
             "ForInStatement",
+            "ForOfStatement",
             "ForStatement",
             "FunctionDeclaration",
             "IfStatement",
@@ -490,6 +589,7 @@
         ],
         "Expression": [
             "ArrayExpression",
+            "ArrowFunctionExpression",
             "AssignmentExpression",
             "BinaryExpression",
             "CallExpression",
@@ -504,7 +604,32 @@
             "SequenceExpression",
             "ThisExpression",
             "UnaryExpression",
-            "UpdateExpression"
+            "UpdateExpression",
+            "YieldExpression"
+        ],
+        "ImportDeclarationSpecifier": [
+            "ImportSpecifier",
+            "ImportDefaultSpecifier",
+            "ImportNamespaceSpecifier"
+        ],
+        "ModuleItem": [
+            "ImportOrExportDeclaration",
+            "Statement"
+        ],
+        "ImportOrExportDeclaration": [
+            "ImportDeclaration"
+        ],
+        "ExpressionOrSuper": [
+            "Expression",
+            "Super"
+        ],
+        "ExpressionOrSpread": [
+            "Expression",
+            "SpreadElement"
+        ],
+        "FunctionBody": [
+            "BlockStatement",
+            "Expression"
         ],
         "Pattern": [
             "Identifier"
@@ -591,6 +716,10 @@
             "And": "&&",
             "NullCoalescing": "??",
             "Or": "||"
+        },
+        "SourceType": {
+            "Script": "script",
+            "Module": "module"
         }
     }
 }

compiler/forget/crates/estree/src/generated.rs

@@ -16,7 +16,13 @@ pub struct Position {
 pub struct Function {
     id: Option<Identifier>,
     params: Vec<Pattern>,
-    body: Option<BlockStatement>,
+    body: Option<FunctionBody>,
+    #[serde(rename = "generator")]
+    #[serde(default)]
+    is_generator: bool,
+    #[serde(rename = "async")]
+    #[serde(default)]
+    is_async: bool,
 }
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct RegExpValue {
@@ -47,7 +53,9 @@ pub struct Literal {
 }
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Program {
-    body: Vec<Statement>,
+    body: Vec<ModuleItem>,
+    #[serde(rename = "sourceType")]
+    source_type: Option<SourceType>,
     #[serde(default)]
     pub loc: Option<SourceLocation>,
     #[serde(default)]
@@ -57,6 +65,8 @@ pub struct Program {
 pub struct ExpressionStatement {
     expression: Expression,
     #[serde(default)]
+    directive: Option<String>,
+    #[serde(default)]
     pub loc: Option<SourceLocation>,
     #[serde(default)]
     pub range: Option<SourceRange>,
@@ -220,6 +230,16 @@ pub struct ForInStatement {
     pub range: Option<SourceRange>,
 }
 #[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ForOfStatement {
+    left: ForInInit,
+    right: Expression,
+    body: Statement,
+    #[serde(default)]
+    pub loc: Option<SourceLocation>,
+    #[serde(default)]
+    pub range: Option<SourceRange>,
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct FunctionDeclaration {
     #[serde(flatten)]
     function: Function,
@@ -255,7 +275,7 @@ pub struct ThisExpression {
 }
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct ArrayExpression {
-    elements: Vec<Option<Expression>>,
+    elements: Vec<Option<ExpressionOrSpread>>,
     #[serde(default)]
     pub loc: Option<SourceLocation>,
     #[serde(default)]
@@ -289,6 +309,17 @@ pub struct FunctionExpression {
     pub range: Option<SourceRange>,
 }
 #[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ArrowFunctionExpression {
+    #[serde(flatten)]
+    function: Function,
+    #[serde(rename = "expression")]
+    is_expression: bool,
+    #[serde(default)]
+    pub loc: Option<SourceLocation>,
+    #[serde(default)]
+    pub range: Option<SourceRange>,
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct UnaryExpression {
     operator: UnaryOperator,
     prefix: bool,
@@ -340,7 +371,7 @@ pub struct LogicalExpression {
 }
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct MemberExpression {
-    object: Expression,
+    object: ExpressionOrSuper,
     property: Expression,
     computed: bool,
     #[serde(default)]
@@ -360,8 +391,8 @@ pub struct ConditionalExpression {
 }
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct CallExpression {
-    callee: Expression,
-    arguments: Vec<Expression>,
+    callee: ExpressionOrSuper,
+    arguments: Vec<ExpressionOrSpread>,
     #[serde(default)]
     pub loc: Option<SourceLocation>,
     #[serde(default)]
@@ -370,7 +401,7 @@ pub struct CallExpression {
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct NewExpression {
     callee: Expression,
-    arguments: Vec<Expression>,
+    arguments: Vec<ExpressionOrSpread>,
     #[serde(default)]
     pub loc: Option<SourceLocation>,
     #[serde(default)]
@@ -385,6 +416,66 @@ pub struct SequenceExpression {
     pub range: Option<SourceRange>,
 }
 #[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct Super {
+    #[serde(default)]
+    pub loc: Option<SourceLocation>,
+    #[serde(default)]
+    pub range: Option<SourceRange>,
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct SpreadElement {
+    argument: Expression,
+    #[serde(default)]
+    pub loc: Option<SourceLocation>,
+    #[serde(default)]
+    pub range: Option<SourceRange>,
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct YieldExpression {
+    #[serde(default)]
+    argument: Option<Expression>,
+    #[serde(rename = "delegate")]
+    is_delegate: bool,
+    #[serde(default)]
+    pub loc: Option<SourceLocation>,
+    #[serde(default)]
+    pub range: Option<SourceRange>,
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ImportDeclaration {
+    specifiers: Vec<ImportDeclarationSpecifier>,
+    source: Literal,
+    #[serde(default)]
+    pub loc: Option<SourceLocation>,
+    #[serde(default)]
+    pub range: Option<SourceRange>,
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ImportSpecifier {
+    imported: Identifier,
+    local: Identifier,
+    #[serde(default)]
+    pub loc: Option<SourceLocation>,
+    #[serde(default)]
+    pub range: Option<SourceRange>,
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ImportDefaultSpecifier {
+    local: Identifier,
+    #[serde(default)]
+    pub loc: Option<SourceLocation>,
+    #[serde(default)]
+    pub range: Option<SourceRange>,
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ImportNamespaceSpecifier {
+    local: Identifier,
+    #[serde(default)]
+    pub loc: Option<SourceLocation>,
+    #[serde(default)]
+    pub range: Option<SourceRange>,
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(tag = "type")]
 pub enum Statement {
     BlockStatement(Box<BlockStatement>),
@@ -395,6 +486,7 @@ pub enum Statement {
     EmptyStatement(Box<EmptyStatement>),
     ExpressionStatement(Box<ExpressionStatement>),
     ForInStatement(Box<ForInStatement>),
+    ForOfStatement(Box<ForOfStatement>),
     ForStatement(Box<ForStatement>),
     FunctionDeclaration(Box<FunctionDeclaration>),
     IfStatement(Box<IfStatement>),
@@ -411,6 +503,7 @@ pub enum Statement {
 #[serde(tag = "type")]
 pub enum Expression {
     ArrayExpression(Box<ArrayExpression>),
+    ArrowFunctionExpression(Box<ArrowFunctionExpression>),
     AssignmentExpression(Box<AssignmentExpression>),
     BinaryExpression(Box<BinaryExpression>),
     CallExpression(Box<CallExpression>),
@@ -426,6 +519,43 @@ pub enum Expression {
     ThisExpression(Box<ThisExpression>),
     UnaryExpression(Box<UnaryExpression>),
     UpdateExpression(Box<UpdateExpression>),
+    YieldExpression(Box<YieldExpression>),
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(tag = "type")]
+pub enum ImportDeclarationSpecifier {
+    ImportDefaultSpecifier(Box<ImportDefaultSpecifier>),
+    ImportNamespaceSpecifier(Box<ImportNamespaceSpecifier>),
+    ImportSpecifier(Box<ImportSpecifier>),
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(untagged)]
+pub enum ModuleItem {
+    ImportOrExportDeclaration(ImportOrExportDeclaration),
+    Statement(Statement),
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(tag = "type")]
+pub enum ImportOrExportDeclaration {
+    ImportDeclaration(Box<ImportDeclaration>),
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(untagged)]
+pub enum ExpressionOrSuper {
+    Expression(Expression),
+    Super(Box<Super>),
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(untagged)]
+pub enum ExpressionOrSpread {
+    Expression(Expression),
+    SpreadElement(Box<SpreadElement>),
+}
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(untagged)]
+pub enum FunctionBody {
+    BlockStatement(Box<BlockStatement>),
+    Expression(Expression),
 }
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(tag = "type")]
@@ -914,3 +1044,42 @@ impl std::str::FromStr for LogicalOperator {
         }
     }
 }
+#[derive(
+    Serialize,
+    Deserialize,
+    Clone,
+    Copy,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Hash,
+    Debug
+)]
+pub enum SourceType {
+    /// module
+    #[serde(rename = "module")]
+    Module,
+    /// script
+    #[serde(rename = "script")]
+    Script,
+}
+impl std::fmt::Display for SourceType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let name = match self {
+            Self::Module => "module",
+            Self::Script => "script",
+        };
+        f.write_str(name)
+    }
+}
+impl std::str::FromStr for SourceType {
+    type Err = ();
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "module" => Ok(Self::Module),
+            "script" => Ok(Self::Script),
+            _ => Err(()),
+        }
+    }
+}

compiler/forget/crates/estree/src/generated_extensions.rs

@@ -0,0 +1,8 @@
+// Manual extensions to generated types
+use crate::SourceType;
+
+impl Default for SourceType {
+    fn default() -> Self {
+        Self::Script
+    }
+}

compiler/forget/crates/estree/src/lib.rs

@@ -1,5 +1,6 @@
 mod binding;
 mod generated;
+mod generated_extensions;
 mod js_value;
 mod range;
 

compiler/forget/crates/estree/src/snapshots/estree__tests__fixtures@for-statement.json.snap

@@ -910,6 +910,7 @@ Output:
                       "end": 84
                     }
                   },
+                  "directive": null,
                   "loc": {
                     "source": null,
                     "start": {
@@ -1014,6 +1015,8 @@ Output:
           "end": 103
         }
       },
+      "generator": false,
+      "async": false,
       "loc": {
         "source": null,
         "start": {
@@ -1031,6 +1034,7 @@ Output:
       }
     }
   ],
+  "sourceType": "script",
   "loc": {
     "source": null,
     "start": {

compiler/forget/crates/estree/src/snapshots/estree__tests__fixtures@import.json.snap

@@ -111,9 +111,9 @@ Input:
 
 Output:
 {
-  "sourceType": "module",
   "body": [
     {
+      "type": "ImportDeclaration",
       "specifiers": [
         {
           "type": "ImportDefaultSpecifier",
@@ -154,8 +154,11 @@ Output:
         }
       ],
       "source": {
-        "value": "react",
+        "value": {
+          "String": "react"
+        },
         "raw": "'react'",
+        "regex": null,
         "loc": {
           "source": null,
           "start": {
@@ -189,7 +192,7 @@ Output:
       }
     }
   ],
-  "comments": [],
+  "sourceType": "module",
   "loc": {
     "source": null,
     "start": {

compiler/forget/crates/estree/src/snapshots/estree__tests__fixtures@simple.json.snap

@@ -196,7 +196,6 @@ Input:
 
 Output:
 {
-  "sourceType": "module",
   "body": [
     {
       "type": "FunctionDeclaration",
@@ -241,10 +240,7 @@ Output:
           }
         }
       ],
-      "generator": false,
-      "async": false,
       "body": {
-        "type": "BlockStatement",
         "body": [
           {
             "type": "ReturnStatement",
@@ -291,7 +287,6 @@ Output:
                 }
               },
               "computed": false,
-              "optional": false,
               "loc": {
                 "source": null,
                 "start": {
@@ -341,6 +336,8 @@ Output:
           "end": 51
         }
       },
+      "generator": false,
+      "async": false,
       "loc": {
         "source": null,
         "start": {
@@ -358,7 +355,7 @@ Output:
       }
     }
   ],
-  "comments": [],
+  "sourceType": null,
   "loc": {
     "source": null,
     "start": {