[rust] Fixture tests for parsing and lowering

Adds a new `fixtures` crate intended for running end-to-end tests of the 
compiler. As we expand the compiler this will eventually match our JS fixture 
setup, where we have .js files as input and produce memoized JS output. 

For now, this does the following: 

* Parses with SWC (omg this was painful to setup) 

* Runs SWC's name resolution, which annotates the SWC ast in-place 

* Convert the SWC ast into our `estree` representation 

* Convert `estree` into `hir` for each top-level function declaration in the 
input program 

* Print the Rust `Debug` view of the resulting HIR 

As a next step i'll add a pretty-printer for the HIR to roughly match what we 
have in JS.
This commit is contained in:
Joe Savona
2023-07-06 09:24:46 +09:00
parent 1d2e7ee747
commit d3b9948b5e
19 changed files with 2130 additions and 109 deletions
+1401 -90
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -2,6 +2,7 @@
members = [
"crates/build-hir",
"crates/fixtures",
"crates/hir",
"crates/swc-demo",
"crates/estree",
@@ -102,7 +102,7 @@ fn lower_statement<'a>(
Statement::EmptyStatement(_) => {
// no-op
}
_ => todo!(),
_ => todo!("Lower {stmt:#?}"),
}
Ok(())
}
@@ -148,7 +148,7 @@ fn lower_expression<'a>(
ExpressionLike::SpreadElement(_) => {
panic!("SpreadElement may not appear in normal expression position")
}
_ => todo!(),
_ => todo!("Lower expr {expr:#?}"),
}
}
@@ -197,7 +197,7 @@ fn lower_primitive<'a>(
LiteralValue::Null => PrimitiveValue::Null,
LiteralValue::Number(value) => PrimitiveValue::Number(f64::from(value).into()),
LiteralValue::String(s) => PrimitiveValue::String(String::from_str_in(&s, &env.allocator)),
_ => todo!(),
_ => todo!("Lower literal {literal:#?}"),
}
}
@@ -58,12 +58,25 @@ impl<'a> Builder<'a> {
blocks: self.completed,
};
assert!(hir.blocks.len() > 0, "initial block count");
reverse_postorder_blocks(&mut hir);
assert!(hir.blocks.len() > 0, "after reverse_postorder_blocks");
remove_unreachable_for_updates(&mut hir);
assert!(hir.blocks.len() > 0, "after remove_unreachable_for_updates");
remove_unreachable_fallthroughs(&mut hir);
assert!(
hir.blocks.len() > 0,
"after remove_unreachable_fallthroughs"
);
remove_unreachable_do_while_statements(&mut hir);
assert!(
hir.blocks.len() > 0,
"after remove_unreachable_do_while_statements"
);
mark_instruction_ids(&mut hir)?;
assert!(hir.blocks.len() > 0, "after mark_instruction_ids");
mark_predecessors(&mut hir);
assert!(hir.blocks.len() > 0, "after mark_predecessors");
Ok(hir)
}
@@ -166,6 +179,7 @@ fn reverse_postorder_blocks<'a>(hir: &mut HIR<'a>) {
}
TerminalValue::ReturnTerminal(..) => { /* no-op */ }
}
postorder.push(block_id);
}
visit(hir.entry, &hir, &mut visited, &mut postorder);
@@ -241,7 +255,9 @@ fn mark_instruction_ids<'a>(hir: &mut HIR<'a>) -> Result<(), Diagnostic> {
let mut visited = HashSet::<(usize, usize)>::new();
for (block_ix, block) in hir.blocks.values_mut().enumerate() {
for (instr_ix, instr) in block.instructions.iter_mut().enumerate() {
invariant(!visited.insert((block_ix, instr_ix)), || ())?;
invariant(visited.insert((block_ix, instr_ix)), || {
format!("Expected bb{block_ix} i{instr_ix} not to have been visited yet")
})?;
instr.id = id_gen.next();
}
block.terminal.id = id_gen.next();
@@ -275,12 +291,13 @@ fn mark_predecessors<'a>(hir: &mut HIR<'a>) {
visit(hir.entry, None, hir, &mut visited);
}
fn invariant<F>(cond: bool, _f: F) -> Result<(), Diagnostic>
fn invariant<F>(cond: bool, f: F) -> Result<(), Diagnostic>
where
F: FnOnce() -> Diagnostic,
F: FnOnce() -> String,
{
if !cond {
panic!("Oops invariant failed");
let msg = f();
panic!("Invariant: {msg}");
}
Ok(())
}
+2 -2
View File
@@ -6,6 +6,6 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
swc_ecma_ast = { version = "0.106.6", features = ["serde", "serde-impl"] }
estree = { path = "../estree" }
swc_common = "0.31.16"
swc = "0.264.8"
swc_core = { version = "0.79.9", features = ["swc_ecma_visit", "__ecma_transforms", "swc_ecma_ast", "swc_common", "swc", "swc_ecma_parser", "__ecma", "__common", "ecma_ast", "__visit", "__parser"] }
+43 -6
View File
@@ -1,12 +1,49 @@
use std::num::NonZeroU32;
use std::{io::stderr, num::NonZeroU32, sync::Arc};
use swc_common::{source_map::Pos, Span};
use swc_ecma_ast::{
AssignOp, BinaryOp, BlockStmt, Decl, Expr, Ident, Lit, ModuleItem, Pat, PatOrExpr, Program,
Stmt, UnaryOp, VarDeclKind, VarDeclOrExpr,
use swc::Compiler;
use swc_core::common::errors::Handler;
use swc_core::common::source_map::Pos;
use swc_core::common::{FileName, FilePathMapping, Mark, SourceMap, Span, GLOBALS};
use swc_core::ecma::ast::{
AssignOp, BinaryOp, BlockStmt, Decl, EsVersion, Expr, Ident, Lit, ModuleItem, Pat, PatOrExpr,
Program, Stmt, UnaryOp, VarDeclKind, VarDeclOrExpr,
};
use swc_core::ecma::parser::Syntax;
use swc_core::ecma::transforms::base::resolver;
use swc_core::ecma::visit::FoldWith;
pub fn convert_program(program: &Program) -> estree::Program {
/// Parses source text into an estree::Program via SWC, internally performing the parsing
/// and SWC -> ESTree conversion.
pub fn parse(source: &str, file: &str) -> Result<estree::Program, Box<dyn std::error::Error>> {
GLOBALS.set(&Default::default(), || {
let cm = Arc::new(SourceMap::new(FilePathMapping::empty()));
let c = Compiler::new(cm);
let fm =
c.cm.new_source_file(FileName::Real(file.into()), source.to_string());
let handler = Handler::with_emitter_writer(Box::new(stderr()), Some(c.cm.clone()));
let comments = c.comments().clone();
let module = c.parse_js(
fm.clone(),
&handler,
EsVersion::Es5,
Syntax::Typescript(Default::default()),
swc::config::IsModule::Bool(true),
Some(&comments),
)?;
let module = c.run_transform(&handler, false, || {
let unresolved_mark = Mark::new();
let top_level_mark = Mark::new();
module.fold_with(&mut resolver(unresolved_mark, top_level_mark, true))
});
Ok(convert_program(&module))
})
}
fn convert_program(program: &Program) -> estree::Program {
let mut program_items: Vec<estree::ModuleItem>;
match program {
Program::Module(program) => {
@@ -0,0 +1,15 @@
[package]
name = "fixtures"
version = "0.1.0"
edition = "2021"
publish = false
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
insta = "1.30.0"
estree = { path = "../estree" }
estree-swc = { path = "../estree-swc" }
hir = { path = "../hir" }
build-hir = { path = "../build-hir" }
bumpalo = { version = "3.13.0", features = ["collections"] }
@@ -0,0 +1,3 @@
# fixtures
This crate is for tests only, and runs the suite of compiler fixture tests.
@@ -0,0 +1 @@
@@ -0,0 +1,4 @@
function test() {
[true, false, null, 1, 3.14, "hello world!"];
return 2;
}
@@ -0,0 +1,36 @@
use build_hir::build;
use bumpalo::Bump;
use estree::{ModuleItem, Statement};
use estree_swc::parse;
use hir::{Environment, Registry};
use insta::{assert_snapshot, glob};
#[test]
fn fixtures() {
glob!("fixtures/**.js", |path| {
let input = std::fs::read_to_string(path).unwrap();
let ast = parse(&input, path.to_str().unwrap()).unwrap();
let mut output = Vec::new();
for item in ast.body {
if let ModuleItem::Statement(stmt) = item {
if let Statement::FunctionDeclaration(fun) = *stmt {
let allocator = Bump::new();
let environment = allocator.alloc(Environment::new(
&allocator,
hir::Features {
validate_frozen_lambdas: true,
},
Registry,
));
let hir = build(&environment, *fun).unwrap();
output.push(format!("{hir:#?}"));
}
}
}
let joined = output.join("\n\n");
assert_snapshot!(format!("Input:\n{input}\n\nOutput:\n{joined}"));
});
}
@@ -0,0 +1,560 @@
---
source: crates/fixtures/tests/fixtures_test.rs
expression: "format!(\"Input:\\n{input}\\n\\nOutput:\\n{joined}\")"
input_file: crates/fixtures/tests/fixtures/simple.js
---
Input:
function test() {
[true, false, null, 1, 3.14, "hello world!"];
return 2;
}
Output:
Function {
body: HIR {
entry: BlockId(
0,
),
blocks: {
BlockId(
0,
): BasicBlock {
id: BlockId(
0,
),
kind: Block,
instructions: [
Instruction {
id: InstructionId(
0,
),
lvalue: Place {
identifier: Identifier {
id: IdentifierId(
0,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
0,
),
),
},
},
},
effect: None,
},
value: Primitive(
Primitive {
value: Boolean(
true,
),
},
),
},
Instruction {
id: InstructionId(
1,
),
lvalue: Place {
identifier: Identifier {
id: IdentifierId(
1,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
1,
),
),
},
},
},
effect: None,
},
value: Primitive(
Primitive {
value: Boolean(
false,
),
},
),
},
Instruction {
id: InstructionId(
2,
),
lvalue: Place {
identifier: Identifier {
id: IdentifierId(
2,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
2,
),
),
},
},
},
effect: None,
},
value: Primitive(
Primitive {
value: Null,
},
),
},
Instruction {
id: InstructionId(
3,
),
lvalue: Place {
identifier: Identifier {
id: IdentifierId(
3,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
3,
),
),
},
},
},
effect: None,
},
value: Primitive(
Primitive {
value: Number(
Number(
4607182418800017408,
),
),
},
),
},
Instruction {
id: InstructionId(
4,
),
lvalue: Place {
identifier: Identifier {
id: IdentifierId(
4,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
4,
),
),
},
},
},
effect: None,
},
value: Primitive(
Primitive {
value: Number(
Number(
4614253070214989087,
),
),
},
),
},
Instruction {
id: InstructionId(
5,
),
lvalue: Place {
identifier: Identifier {
id: IdentifierId(
5,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
5,
),
),
},
},
},
effect: None,
},
value: Primitive(
Primitive {
value: String(
"hello world!",
),
},
),
},
Instruction {
id: InstructionId(
6,
),
lvalue: Place {
identifier: Identifier {
id: IdentifierId(
6,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
6,
),
),
},
},
},
effect: None,
},
value: Array(
Array {
elements: [
Place(
Place {
identifier: Identifier {
id: IdentifierId(
0,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
0,
),
),
},
},
},
effect: None,
},
),
Place(
Place {
identifier: Identifier {
id: IdentifierId(
1,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
1,
),
),
},
},
},
effect: None,
},
),
Place(
Place {
identifier: Identifier {
id: IdentifierId(
2,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
2,
),
),
},
},
},
effect: None,
},
),
Place(
Place {
identifier: Identifier {
id: IdentifierId(
3,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
3,
),
),
},
},
},
effect: None,
},
),
Place(
Place {
identifier: Identifier {
id: IdentifierId(
4,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
4,
),
),
},
},
},
effect: None,
},
),
Place(
Place {
identifier: Identifier {
id: IdentifierId(
5,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
5,
),
),
},
},
},
effect: None,
},
),
],
},
),
},
Instruction {
id: InstructionId(
7,
),
lvalue: Place {
identifier: Identifier {
id: IdentifierId(
7,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
7,
),
),
},
},
},
effect: None,
},
value: Primitive(
Primitive {
value: Number(
Number(
4611686018427387904,
),
),
},
),
},
],
terminal: Terminal {
id: InstructionId(
8,
),
value: ReturnTerminal(
ReturnTerminal {
value: Place {
identifier: Identifier {
id: IdentifierId(
7,
),
name: None,
data: RefCell {
value: IdentifierData {
mutable_range: MutableRange {
start: InstructionId(
0,
),
end: InstructionId(
0,
),
},
scope: None,
type_: Var(
TypeVarId(
7,
),
),
},
},
},
effect: None,
},
},
),
},
predecessors: {},
},
},
},
is_async: false,
is_generator: false,
}
@@ -8,6 +8,7 @@ use crate::{id_types::BlockId, Instruction, Terminal};
/// continue.
///
/// [1] Assuming no exceptions are thrown.
#[derive(Debug)]
pub struct BasicBlock<'a> {
/// The identifier for the block
pub id: BlockId,
@@ -28,6 +29,7 @@ pub struct BasicBlock<'a> {
pub predecessors: HashSet<BlockId>,
}
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub enum BlockKind {
Block,
Value,
@@ -3,6 +3,7 @@ use indexmap::IndexMap;
use crate::{BasicBlock, BlockId};
/// Represents either a React function or a function expression
#[derive(Debug)]
pub struct Function<'a> {
pub body: HIR<'a>,
pub is_async: bool,
@@ -13,6 +14,7 @@ pub struct Function<'a> {
/// Blocks are stored in reverse postorder (predecessors before successors)
/// so that compiler passes can complete forward data flow analysis in a
/// single pass over the CFG in the case where there are no loops.
#[derive(Debug)]
pub struct HIR<'a> {
/// The id of the first block
pub entry: BlockId,
@@ -39,6 +39,7 @@ impl TypeVarId {
///
/// TODO: rename to more clearly indicate that this is for sequencing
/// and to reflect that it is applied to terminals as well
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
pub struct InstructionId(pub(crate) u32);
pub struct InstructionIdGenerator(u32);
@@ -56,10 +57,13 @@ impl InstructionIdGenerator {
}
/// Uniquely identifies a reactive scope
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
pub struct ScopeId(pub(crate) u32);
/// Uniquely identifiers a builtin function type in the type registry
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
pub struct FunctionId(pub(crate) u32);
/// Uniquely identifiers a builtin object type in the type registry
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
pub struct ObjectId(pub(crate) u32);
+19 -3
View File
@@ -4,12 +4,14 @@ use bumpalo::collections::{String, Vec};
use crate::{IdentifierId, InstructionId, ScopeId, Type};
#[derive(Debug)]
pub struct Instruction<'a> {
pub id: InstructionId,
pub lvalue: Place<'a>,
pub value: InstructionValue<'a>,
}
#[derive(Debug)]
pub enum InstructionValue<'a> {
Array(Array<'a>),
// Await(Await<'a>),
@@ -46,19 +48,23 @@ pub enum InstructionValue<'a> {
// Unsupported(Unsupported<'a>),
}
#[derive(Debug)]
pub struct Array<'a> {
pub elements: Vec<'a, ArrayElement<'a>>,
}
#[derive(Debug)]
pub enum ArrayElement<'a> {
Place(Place<'a>),
Spread(Place<'a>),
}
#[derive(Debug)]
pub struct Primitive<'a> {
pub value: PrimitiveValue<'a>,
}
#[derive(Debug)]
pub enum PrimitiveValue<'a> {
Boolean(bool),
Null,
@@ -84,38 +90,45 @@ impl From<Number> for f64 {
}
}
#[derive(Debug)]
pub struct LoadLocal<'a> {
pub place: Place<'a>,
}
#[derive(Debug)]
pub struct LoadContext<'a> {
pub place: Place<'a>,
}
#[derive(Debug)]
pub struct DeclareLocal<'a> {
pub lvalue: LValue<'a>,
}
#[derive(Debug)]
pub struct DeclareContext<'a> {
pub lvalue: LValue<'a>, // note: kind must be InstructionKind::Let
}
#[derive(Debug)]
pub struct StoreLocal<'a> {
pub lvalue: LValue<'a>,
pub value: Place<'a>,
}
#[derive(Clone)]
#[derive(Clone, Debug)]
pub struct Place<'a> {
pub identifier: Identifier<'a>,
pub effect: Option<Effect>,
}
#[derive(Debug)]
pub struct LValue<'a> {
pub place: Place<'a>,
pub kind: InstructionKind,
}
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub enum InstructionKind {
/// `const` declaration
Const,
@@ -127,7 +140,7 @@ pub enum InstructionKind {
Reassign,
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub enum Effect {
/// This reference freezes the value (corresponds to a place where codegen should emit a freeze instruction)
Freeze,
@@ -163,7 +176,7 @@ impl Effect {
}
}
#[derive(Clone)]
#[derive(Clone, Debug)]
pub struct Identifier<'a> {
/// Uniquely identifiers this identifier
pub id: IdentifierId,
@@ -172,6 +185,7 @@ pub struct Identifier<'a> {
pub data: Rc<RefCell<IdentifierData>>,
}
#[derive(Debug)]
pub struct IdentifierData {
pub mutable_range: MutableRange,
@@ -185,6 +199,7 @@ pub struct IdentifierData {
///
/// Start is inclusive, end is exclusive (ie end is the "first" instruction
/// for which the value is not mutable).
#[derive(Clone, Debug)]
pub struct MutableRange {
/// start of the range, inclusive.
pub start: InstructionId,
@@ -208,6 +223,7 @@ impl Default for MutableRange {
}
}
#[derive(Clone, Debug)]
pub struct ReactiveScope {
pub id: ScopeId,
pub range: MutableRange,
+8 -1
View File
@@ -2,11 +2,13 @@ use crate::{instruction::Place, BlockId, InstructionId};
/// Terminals represent statements or expressions that affect control flow,
/// such as for-of, if-else, return, logical (??), ternaries (?:), etc.
#[derive(Debug)]
pub struct Terminal<'a> {
pub id: InstructionId,
pub value: TerminalValue<'a>,
}
#[derive(Debug)]
pub enum TerminalValue<'a> {
// BranchTerminal(BranchTerminal),
DoWhileTerminal(DoWhileTerminal),
@@ -69,23 +71,26 @@ impl<'a> TerminalValue<'a> {
}
}
#[derive(Debug)]
pub struct GotoTerminal {
pub block: BlockId,
pub kind: GotoKind,
}
#[derive(Clone, Copy)]
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub enum GotoKind {
Break,
Continue,
}
#[derive(Debug)]
pub struct DoWhileTerminal {
pub body: BlockId,
pub test: BlockId,
pub fallthrough: BlockId,
}
#[derive(Debug)]
pub struct IfTerminal<'a> {
pub test: Place<'a>,
pub consequent: BlockId,
@@ -93,10 +98,12 @@ pub struct IfTerminal<'a> {
pub fallthrough: Option<BlockId>,
}
#[derive(Debug)]
pub struct ReturnTerminal<'a> {
pub value: Place<'a>,
}
#[derive(Debug)]
pub struct ForTerminal {
pub init: BlockId,
pub test: BlockId,
+2
View File
@@ -1,5 +1,6 @@
use crate::{FunctionId, ObjectId, TypeVarId};
#[derive(Debug)]
pub enum Type {
Builtin(BuiltinType),
// Phi(Box<PhiType>),
@@ -8,6 +9,7 @@ pub enum Type {
// Prop(Box<PropType>),
}
#[derive(Debug)]
pub enum BuiltinType {
Primitive,
Function(Option<FunctionId>),
+3
View File
@@ -0,0 +1,3 @@
[toolchain]
# Sigh, because SWC
channel = "nightly"