From 7e990419c1314252ba78d2b0fa14c8dddaab72d4 Mon Sep 17 00:00:00 2001
From: Mofei Zhang <mofei.zhang@utexas.edu>
Date: Wed, 3 May 2023 14:50:25 -0400
Subject: [PATCH] [snap tester] Handle unexpected errors by failing fixture

---
 .../packages/snap/src/compiler-worker.ts      | 38 +++++++++++++++----
 compiler/forget/packages/snap/src/runner.ts   | 25 ++++++++++--
 2 files changed, 51 insertions(+), 12 deletions(-)

diff --git a/compiler/forget/packages/snap/src/compiler-worker.ts b/compiler/forget/packages/snap/src/compiler-worker.ts
index ed70062e6b..df657766e3 100644
--- a/compiler/forget/packages/snap/src/compiler-worker.ts
+++ b/compiler/forget/packages/snap/src/compiler-worker.ts
@@ -23,6 +23,7 @@ export type TestResult = {
   outputPath: string;
   actual: string | null; // null == input did not exist
   expected: string | null; // null == output did not exist
+  unexpectedError: string | null;
 };
 
 export async function compile(
@@ -56,6 +57,7 @@ export async function compile(
       outputPath,
       actual: null,
       expected,
+      unexpectedError: null,
     };
   }
 
@@ -127,10 +129,13 @@ export async function compile(
   const expectError = fixture.startsWith("error.");
   if (expectError) {
     if (error === null) {
-      console.log(code);
-      throw new Error(
-        `Expected an error to be thrown for fixture: '${fixture}', remove the 'error.' prefix if an error is not expected.`
-      );
+      return {
+        inputPath,
+        outputPath,
+        actual: code,
+        expected,
+        unexpectedError: `Expected an error to be thrown for fixture: '${fixture}', remove the 'error.' prefix if an error is not expected.`,
+      };
     } else if (code != null) {
       output = `${formatOutput(code)}\n${formatErrorOutput(error)}`;
     } else {
@@ -138,11 +143,22 @@ export async function compile(
     }
   } else {
     if (error !== null) {
-      error.message = `Expected fixture '${fixture}' to succeed but it failed with error:\n\n${error.message}`;
-      throw error;
+      return {
+        inputPath,
+        outputPath,
+        actual: code,
+        expected,
+        unexpectedError: `Expected fixture '${fixture}' to succeed but it failed with error:\n\n${error.message}`,
+      };
     }
     if (code == null || code.length === 0) {
-      throw new Error(`Expected output for fixture '${fixture}'.`);
+      return {
+        inputPath,
+        outputPath,
+        actual: code,
+        expected,
+        unexpectedError: `Expected output for fixture '${fixture}'.`,
+      };
     }
     output = formatOutput(code);
   }
@@ -158,7 +174,13 @@ ${output}
 
   console.error = originalConsoleError;
 
-  return { inputPath, outputPath, actual, expected };
+  return {
+    inputPath,
+    outputPath,
+    actual,
+    expected,
+    unexpectedError: null,
+  };
 }
 
 function formatErrorOutput(error: Error): string {
diff --git a/compiler/forget/packages/snap/src/runner.ts b/compiler/forget/packages/snap/src/runner.ts
index c52bff3294..1dc4d438d9 100644
--- a/compiler/forget/packages/snap/src/runner.ts
+++ b/compiler/forget/packages/snap/src/runner.ts
@@ -181,7 +181,7 @@ async function run(
 function report(results: Results): void {
   const failures: Array<[string, TestResult]> = [];
   for (const [basename, result] of results) {
-    if (result.actual === result.expected) {
+    if (result.actual === result.expected && result.unexpectedError == null) {
       console.log(
         chalk.green.inverse.bold(" PASS ") + " " + chalk.dim(basename)
       );
@@ -196,7 +196,13 @@ function report(results: Results): void {
 
     for (const [basename, result] of failures) {
       console.log(chalk.red.bold("FAIL:") + " " + basename);
-      console.log(diff(result.actual, result.expected) + "\n");
+      if (result.unexpectedError != null) {
+        console.log(
+          ` >> Unexpected error during test: \n${result.unexpectedError}`
+        );
+      } else {
+        console.log(diff(result.actual, result.expected) + "\n");
+      }
     }
   }
 
@@ -214,8 +220,14 @@ async function update(results: Results): Promise<void> {
   let deleted = 0;
   let updated = 0;
   let created = 0;
+  const failed = [];
   for (const [basename, result] of results) {
-    if (result.actual == null) {
+    if (result.unexpectedError != null) {
+      console.log(
+        chalk.red.inverse.bold(" FAILED ") + " " + chalk.dim(basename)
+      );
+      failed.push([basename, result.unexpectedError]);
+    } else if (result.actual == null) {
       // Input was deleted but the expect file still existed, remove it
       console.log(
         chalk.red.inverse.bold(" REMOVE ") + " " + chalk.dim(basename)
@@ -243,7 +255,12 @@ async function update(results: Results): Promise<void> {
       );
     }
   }
-  console.log(`${deleted} Deleted, ${created} Created, ${updated} Updated`);
+  console.log(
+    `${deleted} Deleted, ${created} Created, ${updated} Updated, ${failed.length} Failed`
+  );
+  for (const [basename, errorMsg] of failed) {
+    console.log(`${chalk.red.bold("Fail:")} ${basename}\n${errorMsg}`);
+  }
 }
 
 function watchSrc(