From 52104c6ee30b8fe3e59b6be417e65d2d51cdba52 Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Fri, 22 Sep 2023 05:02:43 -0700 Subject: [PATCH] Update nightlies to bump all the packages in the monorepo and transitive dependencies (#39600) Summary: Pull Request resolved: https://github.com/facebook/react-native/pull/39600 This change will publish all the packages in the monorepo that are not private as during the nightlies, taking also care of keeping the transitive dependencies aligned. ## Changelog: [Internal] - Bump packages and transitive dependencies when doing nightlies. Reviewed By: huntie Differential Revision: D49502330 fbshipit-source-id: 85e2bde13ed2b5dfe33072c9f99f5aaa2c5063ca --- .../get-and-update-nightlies-fixtures.js | 174 ++++++++++++ .../get-and-update-nightlies-test.js | 165 ++++++------ scripts/monorepo/get-and-update-nightlies.js | 249 ++++++++++-------- scripts/npm-utils.js | 5 +- 4 files changed, 396 insertions(+), 197 deletions(-) create mode 100644 scripts/monorepo/__tests__/__fixtures__/get-and-update-nightlies-fixtures.js diff --git a/scripts/monorepo/__tests__/__fixtures__/get-and-update-nightlies-fixtures.js b/scripts/monorepo/__tests__/__fixtures__/get-and-update-nightlies-fixtures.js new file mode 100644 index 00000000000..4fb2738c7e6 --- /dev/null +++ b/scripts/monorepo/__tests__/__fixtures__/get-and-update-nightlies-fixtures.js @@ -0,0 +1,174 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @oncall react-native + */ + +const mockPackages = [ + { + packageManifest: { + name: '@react-native/packageA', + version: 'local-version', + dependencies: {}, + devDependencies: {}, + }, + packageAbsolutePath: '/some/place/packageA', + packageRelativePathFromRoot: './place/packageA', + }, + { + packageManifest: { + name: '@react-native/not_published', + version: 'local-version', + dependencies: {'@react-native/packageA': 'local-version'}, + }, + packageAbsolutePath: '/some/place/not_published', + packageRelativePathFromRoot: './place/not_published', + }, + { + packageManifest: { + name: '@react-native/packageB', + version: 'local-version', + dependencies: {'@react-native/packageA': 'local-version'}, + }, + packageAbsolutePath: '/some/place/packageB', + packageRelativePathFromRoot: './place/packageB', + }, + { + packageManifest: { + name: '@react-native/packageC', + version: 'local-version', + devDependencies: {'@react-native/packageE': 'local-version'}, + }, + packageAbsolutePath: '/some/place/packageC', + packageRelativePathFromRoot: './place/packageC', + }, + { + packageManifest: { + name: '@react-native/packageD', + version: 'local-version', + dependencies: { + '@react-native/packageA': 'local-version', + '@react-native/packageC': 'local-version', + }, + }, + packageAbsolutePath: '/some/place/packageD', + packageRelativePathFromRoot: './place/packageD', + }, + { + packageManifest: {name: '@react-native/packageE', version: 'local-version'}, + packageAbsolutePath: '/some/place/packageE', + packageRelativePathFromRoot: './place/packageE', + }, + { + packageManifest: { + name: '@react-native/packageF', + version: 'local-version', + dependencies: { + '@react-native/packageB': 'local-version', + }, + devDependencies: { + '@react-native/packageD': 'local-version', + }, + }, + packageAbsolutePath: '/some/place/packageF', + packageRelativePathFromRoot: './place/packageF', + }, + { + packageManifest: { + name: '@react-native/packageP', + version: 'local-version', + private: true, + dependencies: {}, + devDependencies: {}, + }, + packageAbsolutePath: '/some/place/packageP', + packageRelativePathFromRoot: './place/packageP', + }, + { + packageManifest: { + name: 'react-native', + version: 'local-version', + private: true, + dependencies: { + '@react-native/packageA': 'local-version', + '@react-native/packageB': 'local-version', + '@react-native/packageC': 'local-version', + }, + devDependencies: { + '@react-native/packageD': 'local-version', + '@react-native/packageE': 'local-version', + '@react-native/packageF': 'local-version', + }, + }, + packageAbsolutePath: '/some/place/react-native', + packageRelativePathFromRoot: './place/react-native', + }, +]; + +function expectedPackageA(newVersion) { + return { + name: '@react-native/packageA', + version: newVersion, + dependencies: {}, + devDependencies: {}, + }; +} + +function expectedPackageB(newVersion) { + return { + name: '@react-native/packageB', + version: newVersion, + dependencies: {'@react-native/packageA': newVersion}, + }; +} + +function expectedPackageC(newVersion) { + return { + name: '@react-native/packageC', + version: newVersion, + devDependencies: {'@react-native/packageE': newVersion}, + }; +} +function expectedPackageD(newVersion) { + return { + name: '@react-native/packageD', + version: newVersion, + dependencies: { + '@react-native/packageA': newVersion, + '@react-native/packageC': newVersion, + }, + }; +} +function expectedPackageE(newVersion) { + return {name: '@react-native/packageE', version: newVersion}; +} +function expectedPackageF(newVersion) { + return { + name: '@react-native/packageF', + version: newVersion, + dependencies: { + '@react-native/packageB': newVersion, + }, + devDependencies: { + '@react-native/packageD': newVersion, + }, + }; +} + +const expectedPackages = { + '@react-native/packageA': expectedPackageA, + '@react-native/packageB': expectedPackageB, + '@react-native/packageC': expectedPackageC, + '@react-native/packageD': expectedPackageD, + '@react-native/packageE': expectedPackageE, + '@react-native/packageF': expectedPackageF, +}; + +module.exports = { + mockPackages, + expectedPackages, +}; diff --git a/scripts/monorepo/__tests__/get-and-update-nightlies-test.js b/scripts/monorepo/__tests__/get-and-update-nightlies-test.js index 5453b0c7344..72654f7431f 100644 --- a/scripts/monorepo/__tests__/get-and-update-nightlies-test.js +++ b/scripts/monorepo/__tests__/get-and-update-nightlies-test.js @@ -8,26 +8,30 @@ */ const getAndUpdateNightlies = require('../get-and-update-nightlies'); +const path = require('path'); +const { + mockPackages, + expectedPackages, +} = require('./__fixtures__/get-and-update-nightlies-fixtures'); -const NPM_NIGHTLY_VERSION = 'published-nightly-version'; -const mockPackages = [ - { - packageManifest: {name: '@react-native/packageA', version: 'local-version'}, - packageAbsolutePath: '/some/place/packageA', - packageRelativePathFromRoot: './place/packageA', - }, -]; - -const execMock = jest.fn(); const writeFileSyncMock = jest.fn(); -const diffPackagesMock = jest.fn(); const publishPackageMock = jest.fn(); +const getPackageVersionStrByTag = jest.fn(); + +function forEachPackageThatShouldBePublished(callback) { + mockPackages.forEach(package => { + if ( + package.packageManifest.name === 'react-native' || + package.packageManifest.private || + package.packageManifest.name === '@react-native/not_published' + ) { + return; + } + callback(package); + }); +} jest - .mock('shelljs', () => ({ - exec: execMock, - rm: jest.fn(), - })) .mock('fs', () => ({ writeFileSync: writeFileSyncMock, })) @@ -41,97 +45,86 @@ jest ), ); }) - .mock('../../scm-utils', () => ({ - restore: jest.fn(), - })) .mock('../../npm-utils', () => ({ - getPackageVersionStrByTag: () => NPM_NIGHTLY_VERSION, - diffPackages: diffPackagesMock, publishPackage: publishPackageMock, - pack: jest.fn(), + getPackageVersionStrByTag: getPackageVersionStrByTag, })); describe('getAndUpdateNightlies', () => { beforeEach(() => { jest.clearAllMocks(); + + getPackageVersionStrByTag.mockImplementation(packageName => { + if (packageName === '@react-native/not_published') { + throw new Error(`Can't find package with name ${packageName}`); + } + return ''; + }); }); - it('publishes because there are changes', () => { + it('Publishes the nightly version', () => { const nightlyVersion = '0.73.0-nightly-202108-shortcommit'; - publishPackageMock.mockImplementationOnce(() => ({code: 0})); - diffPackagesMock.mockImplementationOnce(() => 'some-file-name.js\n'); + publishPackageMock.mockImplementation(() => ({code: 0})); - const latestNightlies = getAndUpdateNightlies(nightlyVersion); + const updatedPackages = getAndUpdateNightlies(nightlyVersion); - // ensure we set the version of the last published nightly (for diffing) - expect(writeFileSyncMock.mock.calls[0][1]).toBe( - '{\n "name": "@react-native/packageA",\n "version": "published-nightly-version"\n}\n', - ); - - expect(diffPackagesMock).toBeCalledWith( - '@react-native/packageA@nightly', - 'react-native-packageA-published-nightly-version.tgz', - { - cwd: '/some/place/packageA', - }, - ); - - // when determining that we DO want to publish, ensure we update the version to the provded nightly version we want to use - expect(writeFileSyncMock.mock.calls[1][1]).toBe( - `{\n "name": "@react-native/packageA",\n "version": "${nightlyVersion}"\n}\n`, - ); - - expect(publishPackageMock).toBeCalled(); - - // Expect the map returned to accurately list the latest nightly version - expect(latestNightlies).toEqual({ - '@react-native/packageA': '0.73.0-nightly-202108-shortcommit', - }); - }); - describe('fails to publish', () => { - let consoleError; - beforeEach(() => { - consoleError = console.error; - console.error = jest.fn(); + expect(writeFileSyncMock).toHaveBeenCalledTimes(6); + forEachPackageThatShouldBePublished(package => { + expect(writeFileSyncMock).toHaveBeenCalledWith( + path.join(package.packageAbsolutePath, 'package.json'), + JSON.stringify( + expectedPackages[package.packageManifest.name](nightlyVersion), + null, + 2, + ) + '\n', + 'utf-8', + ); }); - afterEach(() => { - console.error = consoleError; + expect(publishPackageMock).toHaveBeenCalledTimes(6); + forEachPackageThatShouldBePublished(package => { + expect(publishPackageMock).toHaveBeenCalledWith( + package.packageAbsolutePath, + {otp: undefined, tag: 'nightly'}, + ); }); - it('doesnt update nightly version when fails to publish', () => { - const nightlyVersion = '0.73.0-nightly-202108-shortcommit'; - publishPackageMock.mockImplementationOnce(() => ({ - code: 1, - stderr: 'Some error about it failing to publish', - })); - diffPackagesMock.mockImplementationOnce(() => 'some-file-name.js\n'); - - const latestNightlies = getAndUpdateNightlies(nightlyVersion); - - // Expect the map returned to accurately list the latest nightly version - expect(latestNightlies).toEqual({}); + let expectedResult = {}; + forEachPackageThatShouldBePublished(package => { + expectedResult[package.packageManifest.name] = + package.packageManifest.version; }); + expect(updatedPackages).toEqual(expectedResult); }); - it('doesnt publish because no changes', () => { + it('Throws when a package fails to publish', () => { const nightlyVersion = '0.73.0-nightly-202108-shortcommit'; - diffPackagesMock.mockImplementationOnce(() => '\n'); - - const latestNightlies = getAndUpdateNightlies(nightlyVersion); - - expect(writeFileSyncMock.mock.calls[0][1]).toBe( - '{\n "name": "@react-native/packageA",\n "version": "published-nightly-version"\n}\n', - ); - - // in this test, we expect there to be no differences between last published nightly and local - // so we never update the version and we don't publish - expect(writeFileSyncMock.mock.calls.length).toBe(1); - expect(publishPackageMock).not.toBeCalled(); - - // Since we don't update, we expect the map to list the latest nightly on npm - expect(latestNightlies).toEqual({ - '@react-native/packageA': 'published-nightly-version', + let publishCalls = 0; + publishPackageMock.mockImplementation(() => { + publishCalls += 1; + if (publishCalls === 3) { + return {code: -1}; + } + return {code: 0}; }); + + expect(() => { + getAndUpdateNightlies(nightlyVersion); + }).toThrow(); + + expect(writeFileSyncMock).toHaveBeenCalledTimes(6); + forEachPackageThatShouldBePublished(package => { + expect(writeFileSyncMock).toHaveBeenCalledWith( + path.join(package.packageAbsolutePath, 'package.json'), + JSON.stringify( + expectedPackages[package.packageManifest.name](nightlyVersion), + null, + 2, + ) + '\n', + 'utf-8', + ); + }); + + expect(publishPackageMock).toHaveBeenCalledTimes(3); }); }); diff --git a/scripts/monorepo/get-and-update-nightlies.js b/scripts/monorepo/get-and-update-nightlies.js index c38c79a7053..ec99644e7e5 100644 --- a/scripts/monorepo/get-and-update-nightlies.js +++ b/scripts/monorepo/get-and-update-nightlies.js @@ -4,61 +4,16 @@ * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. * + * @flow * @format * @oncall react_native */ const forEachPackage = require('./for-each-package'); -const {rm} = require('shelljs'); -const { - getPackageVersionStrByTag, - diffPackages, - publishPackage, - pack, -} = require('../npm-utils'); -const {restore} = require('../scm-utils'); +const {publishPackage} = require('../npm-utils'); const path = require('path'); const {writeFileSync} = require('fs'); - -function hasChanges( - currentNightlyVersion, - packageManifest, - packageAbsolutePath, -) { - // Set local package to same nightly version so diff is comparable - writeFileSync( - path.join(packageAbsolutePath, 'package.json'), - JSON.stringify( - {...packageManifest, version: currentNightlyVersion}, - null, - 2, - ) + '\n', - 'utf-8', - ); - - // prepare local package - pack(packageAbsolutePath); - - // ex. react-native-codegen-0.73.0-nightly-20230530-730ca3540.tgz - const localTarballName = `${packageManifest.name - .replace('@', '') - .replace('/', '-')}-${currentNightlyVersion}.tgz`; - - // npm diff --diff=@nightly --diff=. --diff-name-only - const diff = diffPackages( - `${packageManifest.name}@nightly`, - localTarballName, - { - cwd: packageAbsolutePath, - }, - ); - - // delete tarball and restore package.json changes - rm(path.join(packageAbsolutePath, localTarballName)); - restore(packageAbsolutePath); - - return diff.trim().length !== 0; -} +const {getPackageVersionStrByTag} = require('../npm-utils'); /** * Get the latest nightly version of each monorepo package and publishes a new nightly if there have been updates. @@ -67,75 +22,149 @@ function hasChanges( * This is called by `react-native`'s nightly job. * Note: This does not publish `package/react-native`'s nightly. That is handled in `publish-npm`. */ -function getAndUpdateNightlies(nightlyVersion) { - const nightlyVersions = {}; +function getAndUpdateNightlies( + nightlyVersion /*: string */, +) /*: {| [string]: string|}*/ { + const packages = getPackagesToPublish(); + + updateDependencies(packages, nightlyVersion); + + publishPackages(packages); + + return reducePackagesToNameVersion(packages); +} + +/*:: +type PackageMap = {|[string]: PackageMetadata |} +type PackageMetadata = {| + // The absolute path to the package + path: string, + + // The parsed package.json contents + packageJson: Object, +|}; +*/ + +/** + * Extract package metadata from the monorepo + * It skips react-native, the packages marked as private and packages not already published on NPM. + * + * @return Dictionary where the string is the name of the package and the PackageMetadata + * is an object that contains the absolute path to the package and the packageJson. + */ +function getPackagesToPublish() /*: PackageMap */ { + let packages = {}; forEachPackage( (packageAbsolutePath, packageRelativePathFromRoot, packageManifest) => { - if (packageManifest.private) { - console.log(`\u23ED Skipping private package ${packageManifest.name}`); - + if (packageManifest.private || !isPublishedOnNPM(packageManifest.name)) { + console.log( + `\u23F1 Skipping nightly for ${packageManifest.name}. It is either pivate or unpublished`, + ); return; } - - let lastPublishedNightlyVersion; - let shouldPublishNightly = false; - console.log( - `\n---- Attempting to publish nightly for ${packageManifest.name}`, - ); - try { - lastPublishedNightlyVersion = getPackageVersionStrByTag( - packageManifest.name, - 'nightly', - ); - shouldPublishNightly = hasChanges( - lastPublishedNightlyVersion, - packageManifest, - packageAbsolutePath, - ); - } catch (e) { - console.error( - `Unable to verify if ${packageManifest.name} has changes due to error:`, - ); - console.error(e.message); - console.log(`\u23ED Skipping package ${packageManifest.name}`); - return; - } - - if (!shouldPublishNightly) { - console.log( - `Detected no changes in ${packageManifest.name}@nightly since last publish. Skipping.`, - ); - nightlyVersions[packageManifest.name] = lastPublishedNightlyVersion; - return; - } - - // Set the local package to the updated nightly version - writeFileSync( - path.join(packageAbsolutePath, 'package.json'), - JSON.stringify({...packageManifest, version: nightlyVersion}, null, 2) + - '\n', - 'utf-8', - ); - - const result = publishPackage(packageAbsolutePath, { - tag: 'nightly', - otp: process.env.NPM_CONFIG_OTP, - }); - if (result.code !== 0) { - console.log( - `\u274c Failed to publish version ${nightlyVersion} of ${packageManifest.name}. npm publish exited with code ${result.code}:`, - ); - console.error(result.stderr); - } else { - console.log( - `\u2705 Successfully published new version of ${packageManifest.name}`, - ); - nightlyVersions[packageManifest.name] = nightlyVersion; - } + packages[packageManifest.name] = { + path: packageAbsolutePath, + packageJson: packageManifest, + }; }, ); - return nightlyVersions; + + return packages; +} + +function isPublishedOnNPM(packageName /*: string */) /*: boolean */ { + try { + getPackageVersionStrByTag(packageName); + return true; + } catch (e) { + return false; + } +} + +/** + * Update the dependencies in the packages to the passed nightlyVersion. + * The function update the packages dependencies and devDepenencies if they contains other packages in the monorepo. + * The function also writes the updated package.json to disk. + */ +function updateDependencies( + packages /*: PackageMap */, + nightlyVersion /*: string */, +) { + Object.keys(packages).forEach(packageName => { + const package = packages[packageName]; + const packageManifest = package.packageJson; + + if (packageManifest.dependencies) { + for (const dependency of Object.keys(packageManifest.dependencies)) { + if (packages[dependency]) { + // the dependency is in the monorepo + packages[packageName].packageJson.dependencies[dependency] = + nightlyVersion; + } + } + } + + if (packageManifest.devDependencies) { + for (const dependency of Object.keys(packageManifest.devDependencies)) { + if (packages[dependency]) { + // the dependency is in the monorepo + packages[packageName].packageJson.devDependencies[dependency] = + nightlyVersion; + } + } + } + + packages[packageName].packageJson.version = nightlyVersion; + + writeFileSync( + path.join(packages[packageName].path, 'package.json'), + JSON.stringify(packages[packageName].packageJson, null, 2) + '\n', + 'utf-8', + ); + }); +} + +/** + * Publish the passed set of packages to npm with the `nightly` tag. + * In case a package fails to be published, it throws an error, stopping the nightly publishing completely + */ +function publishPackages(packages /*: PackageMap */) { + for (const [packageName, packageMetadata] of Object.entries(packages)) { + const packageAbsolutePath = packageMetadata.path; + const nightlyVersion = packageMetadata.packageJson.version; + + const result = publishPackage(packageAbsolutePath, { + tag: 'nightly', + otp: process.env.NPM_CONFIG_OTP, + }); + + if (result.code !== 0) { + throw new Error( + `\u274c Failed to publish version ${nightlyVersion} of ${packageName}. npm publish exited with code ${result.code}:`, + ); + } + + console.log(`\u2705 Successfully published new version of ${packageName}`); + } +} + +/** + * Reduces the Dictionary to a Dictionary. + * where the key is the name of the package and the value is the version number. + * + * @return Dictionary with package name and the new version number. + */ +function reducePackagesToNameVersion( + packages /*: PackageMap */, +) /*: {| [string]: string |} */ { + return Object.keys(packages).reduce( + (result /*: {| [string]: string |} */, name /*: string */) => { + result[name] = packages[name].packageJson.version; + return result; + }, + {}, + ); } module.exports = getAndUpdateNightlies; diff --git a/scripts/npm-utils.js b/scripts/npm-utils.js index 5e467cd8685..b3890f4d302 100644 --- a/scripts/npm-utils.js +++ b/scripts/npm-utils.js @@ -73,7 +73,10 @@ function getNpmInfo(buildType) { } function getPackageVersionStrByTag(packageName, tag) { - const result = exec(`npm view ${packageName}@${tag} version`, {silent: true}); + const npmString = tag + ? `npm view ${packageName}@${tag} version` + : `npm view ${packageName} version`; + const result = exec(npmString, {silent: true}); if (result.code) { throw new Error(`Failed to get ${tag} version from npm\n${result.stderr}`);