Update nightlies to bump all the packages in the monorepo and transitive dependencies (#39600)

Summary:
Pull Request resolved: https://github.com/facebook/react-native/pull/39600

This change will publish all the packages in the monorepo that are not private as during the nightlies, taking also care of keeping the transitive dependencies aligned.

## Changelog:
[Internal] - Bump packages and transitive dependencies when doing nightlies.

Reviewed By: huntie

Differential Revision: D49502330

fbshipit-source-id: 85e2bde13ed2b5dfe33072c9f99f5aaa2c5063ca

scripts/monorepo/__tests__/__fixtures__/get-and-update-nightlies-fixtures.js

@@ -0,0 +1,174 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall react-native
+ */
+
+const mockPackages = [
+  {
+    packageManifest: {
+      name: '@react-native/packageA',
+      version: 'local-version',
+      dependencies: {},
+      devDependencies: {},
+    },
+    packageAbsolutePath: '/some/place/packageA',
+    packageRelativePathFromRoot: './place/packageA',
+  },
+  {
+    packageManifest: {
+      name: '@react-native/not_published',
+      version: 'local-version',
+      dependencies: {'@react-native/packageA': 'local-version'},
+    },
+    packageAbsolutePath: '/some/place/not_published',
+    packageRelativePathFromRoot: './place/not_published',
+  },
+  {
+    packageManifest: {
+      name: '@react-native/packageB',
+      version: 'local-version',
+      dependencies: {'@react-native/packageA': 'local-version'},
+    },
+    packageAbsolutePath: '/some/place/packageB',
+    packageRelativePathFromRoot: './place/packageB',
+  },
+  {
+    packageManifest: {
+      name: '@react-native/packageC',
+      version: 'local-version',
+      devDependencies: {'@react-native/packageE': 'local-version'},
+    },
+    packageAbsolutePath: '/some/place/packageC',
+    packageRelativePathFromRoot: './place/packageC',
+  },
+  {
+    packageManifest: {
+      name: '@react-native/packageD',
+      version: 'local-version',
+      dependencies: {
+        '@react-native/packageA': 'local-version',
+        '@react-native/packageC': 'local-version',
+      },
+    },
+    packageAbsolutePath: '/some/place/packageD',
+    packageRelativePathFromRoot: './place/packageD',
+  },
+  {
+    packageManifest: {name: '@react-native/packageE', version: 'local-version'},
+    packageAbsolutePath: '/some/place/packageE',
+    packageRelativePathFromRoot: './place/packageE',
+  },
+  {
+    packageManifest: {
+      name: '@react-native/packageF',
+      version: 'local-version',
+      dependencies: {
+        '@react-native/packageB': 'local-version',
+      },
+      devDependencies: {
+        '@react-native/packageD': 'local-version',
+      },
+    },
+    packageAbsolutePath: '/some/place/packageF',
+    packageRelativePathFromRoot: './place/packageF',
+  },
+  {
+    packageManifest: {
+      name: '@react-native/packageP',
+      version: 'local-version',
+      private: true,
+      dependencies: {},
+      devDependencies: {},
+    },
+    packageAbsolutePath: '/some/place/packageP',
+    packageRelativePathFromRoot: './place/packageP',
+  },
+  {
+    packageManifest: {
+      name: 'react-native',
+      version: 'local-version',
+      private: true,
+      dependencies: {
+        '@react-native/packageA': 'local-version',
+        '@react-native/packageB': 'local-version',
+        '@react-native/packageC': 'local-version',
+      },
+      devDependencies: {
+        '@react-native/packageD': 'local-version',
+        '@react-native/packageE': 'local-version',
+        '@react-native/packageF': 'local-version',
+      },
+    },
+    packageAbsolutePath: '/some/place/react-native',
+    packageRelativePathFromRoot: './place/react-native',
+  },
+];
+
+function expectedPackageA(newVersion) {
+  return {
+    name: '@react-native/packageA',
+    version: newVersion,
+    dependencies: {},
+    devDependencies: {},
+  };
+}
+
+function expectedPackageB(newVersion) {
+  return {
+    name: '@react-native/packageB',
+    version: newVersion,
+    dependencies: {'@react-native/packageA': newVersion},
+  };
+}
+
+function expectedPackageC(newVersion) {
+  return {
+    name: '@react-native/packageC',
+    version: newVersion,
+    devDependencies: {'@react-native/packageE': newVersion},
+  };
+}
+function expectedPackageD(newVersion) {
+  return {
+    name: '@react-native/packageD',
+    version: newVersion,
+    dependencies: {
+      '@react-native/packageA': newVersion,
+      '@react-native/packageC': newVersion,
+    },
+  };
+}
+function expectedPackageE(newVersion) {
+  return {name: '@react-native/packageE', version: newVersion};
+}
+function expectedPackageF(newVersion) {
+  return {
+    name: '@react-native/packageF',
+    version: newVersion,
+    dependencies: {
+      '@react-native/packageB': newVersion,
+    },
+    devDependencies: {
+      '@react-native/packageD': newVersion,
+    },
+  };
+}
+
+const expectedPackages = {
+  '@react-native/packageA': expectedPackageA,
+  '@react-native/packageB': expectedPackageB,
+  '@react-native/packageC': expectedPackageC,
+  '@react-native/packageD': expectedPackageD,
+  '@react-native/packageE': expectedPackageE,
+  '@react-native/packageF': expectedPackageF,
+};
+
+module.exports = {
+  mockPackages,
+  expectedPackages,
+};

scripts/monorepo/__tests__/get-and-update-nightlies-test.js

@@ -8,26 +8,30 @@
  */
 
 const getAndUpdateNightlies = require('../get-and-update-nightlies');
+const path = require('path');
+const {
+  mockPackages,
+  expectedPackages,
+} = require('./__fixtures__/get-and-update-nightlies-fixtures');
 
-const NPM_NIGHTLY_VERSION = 'published-nightly-version';
-const mockPackages = [
-  {
-    packageManifest: {name: '@react-native/packageA', version: 'local-version'},
-    packageAbsolutePath: '/some/place/packageA',
-    packageRelativePathFromRoot: './place/packageA',
-  },
-];
-
-const execMock = jest.fn();
 const writeFileSyncMock = jest.fn();
-const diffPackagesMock = jest.fn();
 const publishPackageMock = jest.fn();
+const getPackageVersionStrByTag = jest.fn();
+
+function forEachPackageThatShouldBePublished(callback) {
+  mockPackages.forEach(package => {
+    if (
+      package.packageManifest.name === 'react-native' ||
+      package.packageManifest.private ||
+      package.packageManifest.name === '@react-native/not_published'
+    ) {
+      return;
+    }
+    callback(package);
+  });
+}
 
 jest
-  .mock('shelljs', () => ({
-    exec: execMock,
-    rm: jest.fn(),
-  }))
   .mock('fs', () => ({
     writeFileSync: writeFileSyncMock,
   }))
@@ -41,97 +45,86 @@ jest
         ),
     );
   })
-  .mock('../../scm-utils', () => ({
-    restore: jest.fn(),
-  }))
   .mock('../../npm-utils', () => ({
-    getPackageVersionStrByTag: () => NPM_NIGHTLY_VERSION,
-    diffPackages: diffPackagesMock,
     publishPackage: publishPackageMock,
-    pack: jest.fn(),
+    getPackageVersionStrByTag: getPackageVersionStrByTag,
   }));
 
 describe('getAndUpdateNightlies', () => {
   beforeEach(() => {
     jest.clearAllMocks();
+
+    getPackageVersionStrByTag.mockImplementation(packageName => {
+      if (packageName === '@react-native/not_published') {
+        throw new Error(`Can't find package with name ${packageName}`);
+      }
+      return '';
+    });
   });
 
-  it('publishes because there are changes', () => {
+  it('Publishes the nightly version', () => {
     const nightlyVersion = '0.73.0-nightly-202108-shortcommit';
-    publishPackageMock.mockImplementationOnce(() => ({code: 0}));
-    diffPackagesMock.mockImplementationOnce(() => 'some-file-name.js\n');
+    publishPackageMock.mockImplementation(() => ({code: 0}));
 
-    const latestNightlies = getAndUpdateNightlies(nightlyVersion);
+    const updatedPackages = getAndUpdateNightlies(nightlyVersion);
 
-    // ensure we set the version of the last published nightly (for diffing)
-    expect(writeFileSyncMock.mock.calls[0][1]).toBe(
-      '{\n  "name": "@react-native/packageA",\n  "version": "published-nightly-version"\n}\n',
-    );
-
-    expect(diffPackagesMock).toBeCalledWith(
-      '@react-native/packageA@nightly',
-      'react-native-packageA-published-nightly-version.tgz',
-      {
-        cwd: '/some/place/packageA',
-      },
-    );
-
-    // when determining that we DO want to publish, ensure we update the version to the provded nightly version we want to use
-    expect(writeFileSyncMock.mock.calls[1][1]).toBe(
-      `{\n  "name": "@react-native/packageA",\n  "version": "${nightlyVersion}"\n}\n`,
-    );
-
-    expect(publishPackageMock).toBeCalled();
-
-    // Expect the map returned to accurately list the latest nightly version
-    expect(latestNightlies).toEqual({
-      '@react-native/packageA': '0.73.0-nightly-202108-shortcommit',
-    });
-  });
-  describe('fails to publish', () => {
-    let consoleError;
-    beforeEach(() => {
-      consoleError = console.error;
-      console.error = jest.fn();
+    expect(writeFileSyncMock).toHaveBeenCalledTimes(6);
+    forEachPackageThatShouldBePublished(package => {
+      expect(writeFileSyncMock).toHaveBeenCalledWith(
+        path.join(package.packageAbsolutePath, 'package.json'),
+        JSON.stringify(
+          expectedPackages[package.packageManifest.name](nightlyVersion),
+          null,
+          2,
+        ) + '\n',
+        'utf-8',
+      );
     });
 
-    afterEach(() => {
-      console.error = consoleError;
+    expect(publishPackageMock).toHaveBeenCalledTimes(6);
+    forEachPackageThatShouldBePublished(package => {
+      expect(publishPackageMock).toHaveBeenCalledWith(
+        package.packageAbsolutePath,
+        {otp: undefined, tag: 'nightly'},
+      );
     });
 
-    it('doesnt update nightly version when fails to publish', () => {
-      const nightlyVersion = '0.73.0-nightly-202108-shortcommit';
-      publishPackageMock.mockImplementationOnce(() => ({
-        code: 1,
-        stderr: 'Some error about it failing to publish',
-      }));
-      diffPackagesMock.mockImplementationOnce(() => 'some-file-name.js\n');
-
-      const latestNightlies = getAndUpdateNightlies(nightlyVersion);
-
-      // Expect the map returned to accurately list the latest nightly version
-      expect(latestNightlies).toEqual({});
+    let expectedResult = {};
+    forEachPackageThatShouldBePublished(package => {
+      expectedResult[package.packageManifest.name] =
+        package.packageManifest.version;
     });
+    expect(updatedPackages).toEqual(expectedResult);
   });
 
-  it('doesnt publish because no changes', () => {
+  it('Throws when a package fails to publish', () => {
     const nightlyVersion = '0.73.0-nightly-202108-shortcommit';
-    diffPackagesMock.mockImplementationOnce(() => '\n');
-
-    const latestNightlies = getAndUpdateNightlies(nightlyVersion);
-
-    expect(writeFileSyncMock.mock.calls[0][1]).toBe(
-      '{\n  "name": "@react-native/packageA",\n  "version": "published-nightly-version"\n}\n',
-    );
-
-    // in this test, we expect there to be no differences between last published nightly and local
-    // so we never update the version and we don't publish
-    expect(writeFileSyncMock.mock.calls.length).toBe(1);
-    expect(publishPackageMock).not.toBeCalled();
-
-    // Since we don't update, we expect the map to list the latest nightly on npm
-    expect(latestNightlies).toEqual({
-      '@react-native/packageA': 'published-nightly-version',
+    let publishCalls = 0;
+    publishPackageMock.mockImplementation(() => {
+      publishCalls += 1;
+      if (publishCalls === 3) {
+        return {code: -1};
+      }
+      return {code: 0};
     });
+
+    expect(() => {
+      getAndUpdateNightlies(nightlyVersion);
+    }).toThrow();
+
+    expect(writeFileSyncMock).toHaveBeenCalledTimes(6);
+    forEachPackageThatShouldBePublished(package => {
+      expect(writeFileSyncMock).toHaveBeenCalledWith(
+        path.join(package.packageAbsolutePath, 'package.json'),
+        JSON.stringify(
+          expectedPackages[package.packageManifest.name](nightlyVersion),
+          null,
+          2,
+        ) + '\n',
+        'utf-8',
+      );
+    });
+
+    expect(publishPackageMock).toHaveBeenCalledTimes(3);
   });
 });

scripts/monorepo/get-and-update-nightlies.js

@@ -4,61 +4,16 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  *
+ * @flow
  * @format
  * @oncall react_native
  */
 
 const forEachPackage = require('./for-each-package');
-const {rm} = require('shelljs');
-const {
-  getPackageVersionStrByTag,
-  diffPackages,
-  publishPackage,
-  pack,
-} = require('../npm-utils');
-const {restore} = require('../scm-utils');
+const {publishPackage} = require('../npm-utils');
 const path = require('path');
 const {writeFileSync} = require('fs');
-
-function hasChanges(
-  currentNightlyVersion,
-  packageManifest,
-  packageAbsolutePath,
-) {
-  // Set local package to same nightly version so diff is comparable
-  writeFileSync(
-    path.join(packageAbsolutePath, 'package.json'),
-    JSON.stringify(
-      {...packageManifest, version: currentNightlyVersion},
-      null,
-      2,
-    ) + '\n',
-    'utf-8',
-  );
-
-  // prepare local package
-  pack(packageAbsolutePath);
-
-  // ex. react-native-codegen-0.73.0-nightly-20230530-730ca3540.tgz
-  const localTarballName = `${packageManifest.name
-    .replace('@', '')
-    .replace('/', '-')}-${currentNightlyVersion}.tgz`;
-
-  // npm diff --diff=<package-name>@nightly --diff=. --diff-name-only
-  const diff = diffPackages(
-    `${packageManifest.name}@nightly`,
-    localTarballName,
-    {
-      cwd: packageAbsolutePath,
-    },
-  );
-
-  // delete tarball and restore package.json changes
-  rm(path.join(packageAbsolutePath, localTarballName));
-  restore(packageAbsolutePath);
-
-  return diff.trim().length !== 0;
-}
+const {getPackageVersionStrByTag} = require('../npm-utils');
 
 /**
  * Get the latest nightly version of each monorepo package and publishes a new nightly if there have been updates.
@@ -67,75 +22,149 @@ function hasChanges(
  * This is called by `react-native`'s nightly job.
  * Note: This does not publish `package/react-native`'s nightly. That is handled in `publish-npm`.
  */
-function getAndUpdateNightlies(nightlyVersion) {
-  const nightlyVersions = {};
+function getAndUpdateNightlies(
+  nightlyVersion /*: string */,
+) /*: {| [string]: string|}*/ {
+  const packages = getPackagesToPublish();
+
+  updateDependencies(packages, nightlyVersion);
+
+  publishPackages(packages);
+
+  return reducePackagesToNameVersion(packages);
+}
+
+/*::
+type PackageMap = {|[string]: PackageMetadata |}
+type PackageMetadata = {|
+    // The absolute path to the package
+    path: string,
+
+    // The parsed package.json contents
+    packageJson: Object,
+|};
+*/
+
+/**
+ * Extract package metadata from the monorepo
+ * It skips react-native, the packages marked as private and packages not already published on NPM.
+ *
+ * @return Dictionary<String, PackageMetadata> where the string is the name of the package and the PackageMetadata
+ * is an object that contains the absolute path to the package and the packageJson.
+ */
+function getPackagesToPublish() /*: PackageMap */ {
+  let packages = {};
 
   forEachPackage(
     (packageAbsolutePath, packageRelativePathFromRoot, packageManifest) => {
-      if (packageManifest.private) {
-        console.log(`\u23ED Skipping private package ${packageManifest.name}`);
-
+      if (packageManifest.private || !isPublishedOnNPM(packageManifest.name)) {
+        console.log(
+          `\u23F1 Skipping nightly for ${packageManifest.name}. It is either pivate or unpublished`,
+        );
         return;
       }
-
-      let lastPublishedNightlyVersion;
-      let shouldPublishNightly = false;
-      console.log(
-        `\n---- Attempting to publish nightly for ${packageManifest.name}`,
-      );
-      try {
-        lastPublishedNightlyVersion = getPackageVersionStrByTag(
-          packageManifest.name,
-          'nightly',
-        );
-        shouldPublishNightly = hasChanges(
-          lastPublishedNightlyVersion,
-          packageManifest,
-          packageAbsolutePath,
-        );
-      } catch (e) {
-        console.error(
-          `Unable to verify if ${packageManifest.name} has changes due to error:`,
-        );
-        console.error(e.message);
-        console.log(`\u23ED Skipping package ${packageManifest.name}`);
-        return;
-      }
-
-      if (!shouldPublishNightly) {
-        console.log(
-          `Detected no changes in ${packageManifest.name}@nightly since last publish. Skipping.`,
-        );
-        nightlyVersions[packageManifest.name] = lastPublishedNightlyVersion;
-        return;
-      }
-
-      // Set the local package to the updated nightly version
-      writeFileSync(
-        path.join(packageAbsolutePath, 'package.json'),
-        JSON.stringify({...packageManifest, version: nightlyVersion}, null, 2) +
-          '\n',
-        'utf-8',
-      );
-
-      const result = publishPackage(packageAbsolutePath, {
-        tag: 'nightly',
-        otp: process.env.NPM_CONFIG_OTP,
-      });
-      if (result.code !== 0) {
-        console.log(
-          `\u274c Failed to publish version ${nightlyVersion} of ${packageManifest.name}. npm publish exited with code ${result.code}:`,
-        );
-        console.error(result.stderr);
-      } else {
-        console.log(
-          `\u2705 Successfully published new version of ${packageManifest.name}`,
-        );
-        nightlyVersions[packageManifest.name] = nightlyVersion;
-      }
+      packages[packageManifest.name] = {
+        path: packageAbsolutePath,
+        packageJson: packageManifest,
+      };
     },
   );
-  return nightlyVersions;
+
+  return packages;
+}
+
+function isPublishedOnNPM(packageName /*: string */) /*: boolean */ {
+  try {
+    getPackageVersionStrByTag(packageName);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+/**
+ * Update the dependencies in the packages to the passed nightlyVersion.
+ * The function update the packages dependencies and devDepenencies if they contains other packages in the monorepo.
+ * The function also writes the updated package.json to disk.
+ */
+function updateDependencies(
+  packages /*: PackageMap */,
+  nightlyVersion /*: string */,
+) {
+  Object.keys(packages).forEach(packageName => {
+    const package = packages[packageName];
+    const packageManifest = package.packageJson;
+
+    if (packageManifest.dependencies) {
+      for (const dependency of Object.keys(packageManifest.dependencies)) {
+        if (packages[dependency]) {
+          // the dependency is in the monorepo
+          packages[packageName].packageJson.dependencies[dependency] =
+            nightlyVersion;
+        }
+      }
+    }
+
+    if (packageManifest.devDependencies) {
+      for (const dependency of Object.keys(packageManifest.devDependencies)) {
+        if (packages[dependency]) {
+          // the dependency is in the monorepo
+          packages[packageName].packageJson.devDependencies[dependency] =
+            nightlyVersion;
+        }
+      }
+    }
+
+    packages[packageName].packageJson.version = nightlyVersion;
+
+    writeFileSync(
+      path.join(packages[packageName].path, 'package.json'),
+      JSON.stringify(packages[packageName].packageJson, null, 2) + '\n',
+      'utf-8',
+    );
+  });
+}
+
+/**
+ * Publish the passed set of packages to npm with the `nightly` tag.
+ * In case a package fails to be published, it throws an error, stopping the nightly publishing completely
+ */
+function publishPackages(packages /*: PackageMap */) {
+  for (const [packageName, packageMetadata] of Object.entries(packages)) {
+    const packageAbsolutePath = packageMetadata.path;
+    const nightlyVersion = packageMetadata.packageJson.version;
+
+    const result = publishPackage(packageAbsolutePath, {
+      tag: 'nightly',
+      otp: process.env.NPM_CONFIG_OTP,
+    });
+
+    if (result.code !== 0) {
+      throw new Error(
+        `\u274c Failed to publish version ${nightlyVersion} of ${packageName}. npm publish exited with code ${result.code}:`,
+      );
+    }
+
+    console.log(`\u2705 Successfully published new version of ${packageName}`);
+  }
+}
+
+/**
+ * Reduces the Dictionary<String, PackageMetadata> to a Dictionary<String, String>.
+ * where the key is the name of the package and the value is the version number.
+ *
+ * @return Dictionary<String, String> with package name and the new version number.
+ */
+function reducePackagesToNameVersion(
+  packages /*: PackageMap */,
+) /*: {| [string]: string |} */ {
+  return Object.keys(packages).reduce(
+    (result /*: {| [string]: string |} */, name /*: string */) => {
+      result[name] = packages[name].packageJson.version;
+      return result;
+    },
+    {},
+  );
 }
 
 module.exports = getAndUpdateNightlies;

scripts/npm-utils.js

@@ -73,7 +73,10 @@ function getNpmInfo(buildType) {
 }
 
 function getPackageVersionStrByTag(packageName, tag) {
-  const result = exec(`npm view ${packageName}@${tag} version`, {silent: true});
+  const npmString = tag
+    ? `npm view ${packageName}@${tag} version`
+    : `npm view ${packageName} version`;
+  const result = exec(npmString, {silent: true});
 
   if (result.code) {
     throw new Error(`Failed to get ${tag} version from npm\n${result.stderr}`);