Add auto-generation of TypeScript definitions on build (#38990)

Summary:
Pull Request resolved: https://github.com/facebook/react-native/pull/38990

This PR adds auto-generation of Typescript definitions from Flow source code for packages using the shared monorepo build setup (https://github.com/facebook/react-native/pull/38718).

Today, these are the following Node.js packages:

- `packages/community-cli-plugin`
- `packages/dev-middleware` (⬅️ `emitTypeScriptDefs` enabled)

This also improves emitted Flow definitions (`.js.flow`), by using [`flow-api-translator`](https://www.npmjs.com/package/flow-api-translator) to strip implementations.

**All changes**

- Include `flow-api-translator` and configure this to emit type definitions as part of `yarn build`.
    - Add translation from Flow source to TypeScript definitions (`.d.ts`) adjacent to each built file.
    - Improve emitted Flow definitions (`.js.flow`), by using `flow-api-translator` to strip implementations (previously, source files were copied). The Flow and TS defs now mirror each other.
-  Add `emitFlowDefs` and `emitTypeScriptDefs` options to build config to configure the above.
- Integrate TypeScript compiler to perform program validation on emitted `.d.ts` files.
     - This is based on this guide: https://github.com/microsoft/TypeScript-wiki/blob/main/Using-the-Compiler-API.md#a-minimal-compiler.
- Throw an exception on the `rewritePackageExports` step if a package does not define an `"exports"` field.
- Add minimal `flow-typed` definitions for `typescript` 😄.

**Notes on [`flow-api-translator`](https://www.npmjs.com/package/flow-api-translator)**

This project is experimental but is in a more mature state than when we evaluated it earlier in 2023.
- It's now possible to run this tool on our new Node.js packages, since they are exclusively authored using `import`/`export` syntax (a requirement of the tool).
- As a safety net, we run the TypeScript compiler against the generated program, which will fail the build.

Changelog: [Internal]

Reviewed By: robhogan

Differential Revision: D48312463

fbshipit-source-id: 817edb35f911f52fa987946f2d8fc1a319078c9d

scripts/build/babel/node.config.js

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ * @format
+ * @oncall react_native
+ */
+
+/*::
+import type {BabelCoreOptions} from '@babel/core';
+*/
+
+const TARGET_NODE_VERSION = '18';
+
+const config /*: BabelCoreOptions */ = {
+  presets: [
+    '@babel/preset-flow',
+    [
+      '@babel/preset-env',
+      {
+        targets: {
+          node: TARGET_NODE_VERSION,
+        },
+      },
+    ],
+  ],
+  plugins: [
+    [
+      'transform-define',
+      {
+        'process.env.BUILD_EXCLUDE_BABEL_REGISTER': true,
+      },
+    ],
+    [
+      'minify-dead-code-elimination',
+      {keepFnName: true, keepFnArgs: true, keepClassName: true},
+    ],
+  ],
+};
+
+module.exports = config;

scripts/build/build.js

@@ -12,14 +12,22 @@
 const babel = require('@babel/core');
 const {parseArgs} = require('@pkgjs/parseargs');
 const chalk = require('chalk');
+const translate = require('flow-api-translator');
 const glob = require('glob');
 const micromatch = require('micromatch');
-const fs = require('fs');
+const {promises: fs} = require('fs');
 const path = require('path');
 const prettier = require('prettier');
-const {buildConfig, getBabelConfig} = require('./config');
+const ts = require('typescript');
+const {
+  buildConfig,
+  getBabelConfig,
+  getBuildOptions,
+  getTypeScriptCompilerOptions,
+} = require('./config');
 
-const PACKAGES_DIR /*: string */ = path.resolve(__dirname, '../../packages');
+const REPO_ROOT = path.resolve(__dirname, '../..');
+const PACKAGES_DIR /*: string */ = path.join(REPO_ROOT, 'packages');
 const SRC_DIR = 'src';
 const BUILD_DIR = 'dist';
 const JS_FILES_PATTERN = '**/*.js';
@@ -32,7 +40,7 @@ const config = {
   },
 };
 
-function build() {
+async function build() {
   const {
     positionals: packageNames,
     values: {help},
@@ -53,35 +61,48 @@ function build() {
 
   console.log('\n' + chalk.bold.inverse('Building packages') + '\n');
 
-  if (packageNames.length) {
-    packageNames
-      .filter(packageName => packageName in buildConfig.packages)
-      .forEach(buildPackage);
-  } else {
-    Object.keys(buildConfig.packages).forEach(buildPackage);
+  const packagesToBuild = packageNames.length
+    ? packageNames.filter(packageName => packageName in buildConfig.packages)
+    : Object.keys(buildConfig.packages);
+
+  for (const packageName of packagesToBuild) {
+    await buildPackage(packageName);
   }
 
   process.exitCode = 0;
 }
 
-function buildPackage(packageName /*: string */) {
+async function buildPackage(packageName /*: string */) {
+  const {emitTypeScriptDefs} = getBuildOptions(packageName);
   const files = glob.sync(
     path.resolve(PACKAGES_DIR, packageName, SRC_DIR, '**/*'),
     {nodir: true},
   );
-  const packageJsonPath = path.join(PACKAGES_DIR, packageName, 'package.json');
 
   process.stdout.write(
     `${packageName} ${chalk.dim('.').repeat(72 - packageName.length)} `,
   );
-  files.forEach(file => buildFile(path.normalize(file), true));
-  rewritePackageExports(packageJsonPath);
+
+  // Build all files matched for package
+  for (const file of files) {
+    await buildFile(path.normalize(file), true);
+  }
+
+  // Validate program for emitted .d.ts files
+  if (emitTypeScriptDefs) {
+    validateTypeScriptDefs(packageName);
+  }
+
+  // Rewrite package.json "exports" field (src -> dist)
+  await rewritePackageExports(packageName);
+
   process.stdout.write(chalk.reset.inverse.bold.green(' DONE ') + '\n');
 }
 
-function buildFile(file /*: string */, silent /*: boolean */ = false) {
+async function buildFile(file /*: string */, silent /*: boolean */ = false) {
   const packageName = getPackageName(file);
   const buildPath = getBuildPath(file);
+  const {emitFlowDefs, emitTypeScriptDefs} = getBuildOptions(packageName);
 
   const logResult = ({copied, desc} /*: {copied: boolean, desc?: string} */) =>
     silent ||
@@ -97,24 +118,43 @@ function buildFile(file /*: string */, silent /*: boolean */ = false) {
     return;
   }
 
-  fs.mkdirSync(path.dirname(buildPath), {recursive: true});
+  await fs.mkdir(path.dirname(buildPath), {recursive: true});
 
   if (!micromatch.isMatch(file, JS_FILES_PATTERN)) {
-    fs.copyFileSync(file, buildPath);
+    await fs.copyFile(file, buildPath);
     logResult({copied: true, desc: 'copy'});
-  } else {
-    const transformed = prettier.format(
-      babel.transformFileSync(file, getBabelConfig(packageName)).code,
-      {parser: 'babel'},
-    );
-    fs.writeFileSync(buildPath, transformed);
-
-    if (/@flow/.test(fs.readFileSync(file, 'utf-8'))) {
-      fs.copyFileSync(file, buildPath + '.flow');
-    }
-
-    logResult({copied: true});
+    return;
   }
+
+  const source = await fs.readFile(file, 'utf-8');
+  const prettierConfig = {parser: 'babel'};
+
+  // Transform source file using Babel
+  const transformed = prettier.format(
+    (await babel.transformFileAsync(file, getBabelConfig(packageName))).code,
+    prettierConfig,
+  );
+  await fs.writeFile(buildPath, transformed);
+
+  // Translate source Flow types for each type definition target
+  if (/@flow/.test(source)) {
+    await Promise.all([
+      emitFlowDefs
+        ? fs.writeFile(
+            buildPath + '.flow',
+            await translate.translateFlowToFlowDef(source, prettierConfig),
+          )
+        : null,
+      emitTypeScriptDefs
+        ? fs.writeFile(
+            buildPath.replace(/\.js$/, '') + '.d.ts',
+            await translate.translateFlowToTSDef(source, prettierConfig),
+          )
+        : null,
+    ]);
+  }
+
+  logResult({copied: true});
 }
 
 function getPackageName(file /*: string */) /*: string */ {
@@ -130,16 +170,22 @@ function getBuildPath(file /*: string */) /*: string */ {
   );
 }
 
-function rewritePackageExports(packageJsonPath /*: string */) {
-  const pkg = JSON.parse(fs.readFileSync(packageJsonPath, {encoding: 'utf8'}));
+async function rewritePackageExports(packageName /*: string */) {
+  const packageJsonPath = path.join(PACKAGES_DIR, packageName, 'package.json');
+  const pkg = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
 
   if (pkg.exports == null) {
-    return;
+    throw new Error(
+      packageName +
+        ' does not define an "exports" field in its package.json. As part ' +
+        'of the build setup, this field must be used in order to rewrite ' +
+        'paths to built files in production.',
+    );
   }
 
   pkg.exports = rewriteExportsField(pkg.exports);
 
-  fs.writeFileSync(
+  await fs.writeFile(
     packageJsonPath,
     prettier.format(JSON.stringify(pkg), {parser: 'json'}),
   );
@@ -173,6 +219,49 @@ function rewriteExportsTarget(target /*: string */) /*: string */ {
   return target.replace('./' + SRC_DIR + '/', './' + BUILD_DIR + '/');
 }
 
+function validateTypeScriptDefs(packageName /*: string */) {
+  const files = glob.sync(
+    path.resolve(PACKAGES_DIR, packageName, BUILD_DIR, '**/*.d.ts'),
+  );
+  const compilerOptions = {
+    ...getTypeScriptCompilerOptions(packageName),
+    noEmit: true,
+    skipLibCheck: false,
+  };
+  const program = ts.createProgram(files, compilerOptions);
+  const emitResult = program.emit();
+
+  if (emitResult.diagnostics.length) {
+    for (const diagnostic of emitResult.diagnostics) {
+      if (diagnostic.file != null) {
+        let {line, character} = ts.getLineAndCharacterOfPosition(
+          diagnostic.file,
+          diagnostic.start,
+        );
+        let message = ts.flattenDiagnosticMessageText(
+          diagnostic.messageText,
+          '\n',
+        );
+        console.log(
+          // $FlowIssue[incompatible-use] Type refined above
+          `${diagnostic.file.fileName} (${line + 1},${
+            character + 1
+          }): ${message}`,
+        );
+      } else {
+        console.log(
+          ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n'),
+        );
+      }
+    }
+
+    throw new Error(
+      'Failing build because TypeScript errors were encountered for ' +
+        'generated type definitions.',
+    );
+  }
+}
+
 module.exports = {
   buildFile,
   getBuildPath,
@@ -182,5 +271,6 @@ module.exports = {
 };
 
 if (require.main === module) {
-  build();
+  // eslint-disable-next-line no-void
+  void build();
 }

scripts/build/config.js

@@ -11,57 +11,82 @@
 
 /*::
 import type {BabelCoreOptions} from '@babel/core';
+*/
 
+const {ModuleResolutionKind} = require('typescript');
+
+/*::
 export type BuildOptions = $ReadOnly<{
+  // The target runtime to compile for.
   target: 'node',
+
+  // Whether to emit Flow definition files (.js.flow) (default: true).
+  emitFlowDefs?: boolean,
+
+  // Whether to emit TypeScript definition files (.d.ts) (default: false).
+  emitTypeScriptDefs?: boolean,
 }>;
 
 export type BuildConfig = $ReadOnly<{
+  // The packages to include for build and their build options.
   packages: $ReadOnly<{[packageName: string]: BuildOptions}>,
 }>;
 */
 
-const TARGET_NODE_VERSION = '18';
-
+/**
+ * - BUILD CONFIG -
+ *
+ * Add packages here to configure them as part of the monorepo `yarn build`
+ * setup. These must use a consistent package structure and (today) target
+ * Node.js packages only.
+ */
 const buildConfig /*: BuildConfig */ = {
-  // The packages to include for build and their build options
   packages: {
-    'community-cli-plugin': {target: 'node'},
-    'dev-middleware': {target: 'node'},
+    'community-cli-plugin': {
+      target: 'node',
+    },
+    'dev-middleware': {
+      target: 'node',
+      emitTypeScriptDefs: true,
+    },
   },
 };
 
+const defaultBuildOptions = {
+  emitFlowDefs: true,
+  emitTypeScriptDefs: false,
+};
+
+function getBuildOptions(
+  packageName /*: $Keys<BuildConfig['packages']> */,
+) /*: Required<BuildOptions> */ {
+  return {
+    ...defaultBuildOptions,
+    ...buildConfig.packages[packageName],
+  };
+}
+
 function getBabelConfig(
   packageName /*: $Keys<BuildConfig['packages']> */,
 ) /*: BabelCoreOptions */ {
-  const {target} = buildConfig.packages[packageName];
+  const {target} = getBuildOptions(packageName);
+
+  switch (target) {
+    case 'node':
+      return require('./babel/node.config.js');
+  }
+}
+
+function getTypeScriptCompilerOptions(
+  packageName /*: $Keys<BuildConfig['packages']> */,
+) /*: Object */ {
+  const {target} = getBuildOptions(packageName);
 
   switch (target) {
     case 'node':
       return {
-        presets: [
-          '@babel/preset-flow',
-          [
-            '@babel/preset-env',
-            {
-              targets: {
-                node: TARGET_NODE_VERSION,
-              },
-            },
-          ],
-        ],
-        plugins: [
-          [
-            'transform-define',
-            {
-              'process.env.BUILD_EXCLUDE_BABEL_REGISTER': true,
-            },
-          ],
-          [
-            'minify-dead-code-elimination',
-            {keepFnName: true, keepFnArgs: true, keepClassName: true},
-          ],
-        ],
+        ...require('@tsconfig/node18/tsconfig.json').compilerOptions,
+        moduleResolution: ModuleResolutionKind.NodeJs,
       };
   }
 }
@@ -69,4 +94,6 @@ function getBabelConfig(
 module.exports = {
   buildConfig,
   getBabelConfig,
+  getBuildOptions,
+  getTypeScriptCompilerOptions,
 };