add RNTester-E2E: tests for iOS and Android via Appium, WDIO and Jest (#36267)

Summary:
The motivation is to create an E2E testing solution for the RNTester app that can be used to flag when things break and the release crew can use to validate more of the codebase ahead of a release.

This first PR wants to just showcase the main flow (how tests are made, where the E2E folder lives, how it's used by CircleCI) and then we can build on top of it with subsequent PRs, with the help of an umbrella issue to get the community involved in making more tests!

This work is being done cooperation with Callstack developers (mateuszm22 , adzironman, and others) - reason why the branch [lives in a fork](https://github.com/mateuszm22/react-native/tree/k%2Bm/new-rn-tester-E2E).

### [Read the RFC for more details ](https://github.com/react-native-community/discussions-and-proposals/pull/684)

### Extra notes

We are still on WebDriverIO 7 because during the exploratory phase, WDIO was a bit problematic and the workarounds needed to make it work don't seem very nice - see this PR: https://github.com/kelset/react-native-e2e-jest-appium-webdriverio/pull/4 --- this will be revisited in the future, once there's a better path for upgrading.

### Things that will be handled as follow up work

* upgrade to WDIO 8
* [an automated yarn run-e2e-test script](https://github.com/facebook/react-native/pull/36267#discussion_r1269378065)
* [a small refactoring into a js script](https://github.com/facebook/react-native/pull/36267#discussion_r1272050735)

## Changelog:

[INTERNAL] [ADDED] - Added first working configuration for e2e testing

Pull Request resolved: https://github.com/facebook/react-native/pull/36267

Test Plan:
You can play with this locally by [follow the readme](https://github.com/mateuszm22/react-native/blob/k+m/new-rn-tester-E2E/packages/rn-tester-e2e/README.md).

CircleCI jobs will be added.

Reviewed By: cipolleschi

Differential Revision: D46225747

Pulled By: cortinico

fbshipit-source-id: e5428c439afd84ed3dd219d6ada538263120d7c6
This commit is contained in:
Lorenzo Sciandra
2023-07-25 07:58:09 -07:00
committed by Facebook GitHub Bot
parent 74a1822cb5
commit 3c196fb2f5
16 changed files with 3922 additions and 896 deletions
+134
View File
@@ -6,6 +6,7 @@ version: 2.1
orbs:
win: circleci/windows@2.4.0
android: circleci/android@2.3.0
# -------------------------
# REFERENCES
@@ -724,6 +725,136 @@ jobs:
path: ./reports/junit
# -------------------------
# JOBS: iOS E2E Tests
# -------------------------
test_e2e_ios:
executor: reactnativeios
parameters:
ruby_version:
default: "2.7.7"
description: The version of ruby that must be used
type: string
steps:
- checkout_code_with_cache
- run_yarn
- attach_workspace:
at: .
- run:
name: Install appium
command: npm install appium@2.0.0 -g
- run:
name: Install appium drivers
command: |
appium driver install uiautomator2
appium driver install xcuitest
- run:
name: Start Appium server
command: appium --base-path /wd/hub
background: true
- run:
name: Start Metro
command: |
cd packages/rn-tester
yarn start
background: true
- brew_install:
package: cmake
- setup_ruby:
ruby_version: << parameters.ruby_version >>
- run:
name: Install Bundler
command: |
cd packages/rn-tester
bundle check || bundle install
bundle exec pod setup
bundle exec pod install --verbose
- run:
name: Boot iOS Simulator
command: source scripts/.tests.env && xcrun simctl boot "$IOS_DEVICE" || true
- run:
name: Build app
command: |
xcodebuild build \
-workspace packages/rn-tester/RNTesterPods.xcworkspace \
-configuration Debug \
-scheme RNTester \
-sdk iphonesimulator \
-derivedDataPath /tmp/e2e/
- run:
name: Move app to correct directory
command: mv /tmp/e2e/Build/Products/Debug-iphonesimulator/RNTester.app packages/rn-tester-e2e/apps/rn-tester.app
- run:
name: Check Appium server status
command: |
if ! nc -z 127.0.0.1 4723; then
echo Could not find Appium server!
exit 1
fi
- run:
name: Run E2E tests
command: |
cd packages/rn-tester-e2e
yarn test-ios-e2e
# -------------------------
# JOBS: Android E2E Tests
# -------------------------
test_e2e_android:
executor:
name: android/android-machine
tag: 2023.07.1
steps:
- checkout_code_with_cache
- run_yarn
- android/create-avd:
avd-name: e2e_emulator
system-image: system-images;android-33;google_apis;x86_64
install: true
- android/start-emulator:
avd-name: e2e_emulator
no-window: true
restore-gradle-cache-prefix: v1a
post-emulator-launch-assemble-command: ""
- run:
name: Install appium
command: npm install appium@2.0.0 -g
- run:
name: Install appium drivers
command: |
appium driver install uiautomator2
appium driver install xcuitest
- run:
name: Start Appium server
command: appium --base-path /wd/hub
background: true
- run:
name: Start Metro
command: |
cd packages/rn-tester
yarn start
background: true
- attach_workspace:
at: .
- run:
name: Build app
command: |
./gradlew :packages:rn-tester:android:app:assembleHermesDebug -PreactNativeArchitectures=x86_64
- run:
name: Move app to correct directory
command: mv packages/rn-tester/android/app/build/outputs/apk/hermes/debug/app-hermes-x86_64-debug.apk packages/rn-tester-e2e/apps/rn-tester.apk
- run:
name: Check Appium server status
command: |
if ! nc -z 127.0.0.1 4723; then
echo Could not find Appium server
exit 1
fi
- run:
name: Run E2E tests
command: |
cd packages/rn-tester-e2e
yarn test-android-e2e
# -------------------------
# JOBS: Test Android
# -------------------------
test_android:
@@ -1646,6 +1777,9 @@ workflows:
run_disabled_tests: false
- test_android
- test_android_docker_image
- test_e2e_ios:
ruby_version: "2.7.7"
- test_e2e_android
- test_android_template:
requires:
- build_npm_package
+4
View File
@@ -147,3 +147,7 @@ package-lock.json
# Temporary files created by Metro to check the health of the file watcher
.metro-health-check*
# E2E files
/packages/rn-tester-e2e/apps/*.apk
/packages/rn-tester-e2e/apps/*.app
+124
View File
@@ -0,0 +1,124 @@
# RNTester E2E folder
In this folder we have a the setup for running E2E testing in RNTester via the usage of [Appium](https://appium.io/) and [WebDriverIO](https://webdriver.io/) and [Jest](https://jestjs.io/).
- [Setting up locally](#setting-up-locally)
- [(one-off) Setting up Appium](#one-off-setting-up-appium)
- [Building RNTester app](#building-rntester-app)
- [Building for iOS](#building-for-ios)
- [Building for Android](#building-for-android)
- [Setting up the RNTester E2E folder](#setting-up-the-rntester-e2e-folder)
- [Testing the RNTester app E2E](#testing-the-rntester-app-e2e)
- [Adding new tests (and project structure)](#adding-new-tests-and-project-structure)
## Setting up locally
### (one-off) Setting up Appium
The first step you need to do is to ensure to install the tooling:
```bash
npm install appium@2.0.0 -g
appium driver install uiautomator2
appium driver install xcuitest
```
> More details about drivers in Appium [here](https://appium.github.io/appium/docs/en/2.0/guides/managing-exts/) and [here](https://appium.github.io/appium/docs/en/2.0/quickstart/uiauto2-driver/)
You should not need to run install commands for drivers separately more than once, even if you bump the dep in package.json.
### Building RNTester app
Building manually *.app* and *.apk* is required to run automation tests on local environment.
0. *(optional)* If you previously built RNTester, you may need to clean up build files and Pods:
```bash
yarn test-e2e-local-clean && yarn install
```
1. Step 1: install packages for the repository, then navigate in the rn-tester folder
```bash
cd react-native
yarn install
cd packages/rn-tester
```
Now, depending on the platform, there are some specific steps
#### Building for iOS
0. Make sure you have Bundler `gem install bundler` - we use it ensure installing the right version of CocoaPods locally.
1. Install Bundler and CocoaPods dependencies: `bundle install` then `bundle exec pod install` or `yarn setup-ios-hermes` for RNTester with Hermes. In order to use JSC instead of Hermes engine, run: `USE_HERMES=0 bundle exec pod install` or `setup-ios-jsc` instead.
2. You can build app with React Native CLI or manually with Xcode:
1. To build with React Native CLI:
1. Run `npx react-native build-ios --mode Debug --scheme RNTester --buildFolder /path/to/build-folder`, replace `/path/to/build-folder` with the real path.
2. Copy the built app using `mv` - `mv /path/to/build-folder/Build/Products/Debug-iphonesimulator/RNTester.app ~/react-native/packages/rn-tester-e2e/apps` or manually.
2. To build with Xcode, open the generated `RNTester.xcworkspace` and build.
1. Find the **RNTester.app** in `~/Library/Developer/Xcode/DerivedData/RNTesterPods-{id}/Build/Products/Debug-iphonesimulator`
2. Copy the app to the following directory `~/react-native/packages/rn-tester-e2e/apps`.
3. Change its name to: `rn-tester.app`
#### Building for Android
0. You'll need to have all the [prerequisites](https://reactnative.dev/contributing/how-to-build-from-source#prerequisites) (SDK, NDK) for Building React Native installed.
1. Start an Android emulator.
2. Build the app via
```bash
# In order to not use Hermes engine, run `yarn install-android-jsc` instead.
yarn install-android-hermes
yarn start
```
*Note: Building for the first time can take a while.*
3. Find the **app-*-debug.apk** in `~/react-native/packages/rn-tester/android/app/build/outputs/apk/hermes/debug`
4. Copy the app `app-*-debug.apk` to the following directory `~/react-native/packages/rn-tester-e2e/apps`
5. Change its name to: `rn-tester.apk`
### Setting up the RNTester E2E folder
In `react-native/packages/rn-tester-e2e` open the following file
```bash
/react-native/packages/rn-tester-e2e/e2e-config.js
```
And modify lines L24->L39 to reflect your local setup configuration (ex. `platformVersion`, `deviceName`). Make sure to **not** commit this change if you send a PR to add tests.
## Testing the RNTester app E2E
After you have done all the above correctly, and you have the Android/iOS apps in the `rn-tester-e2e/apps` folder, in a dedicated terminal window, run:
```bash
appium --base-path /wd/hub
```
This will start the Appium server - you will need this to keep running.
Then open a second terminal window and start the Metro terminal from the `packages/rn-tester` folder, via `yarn start --reset-cache`. This terminal window also needs to keep running.
Now, make sure that the iOS simulator/the Android emulator is up and running.
Finally, you can open a third terminal window and run:
```bash
yarn test-android-e2e # for android
yarn test-ios-e2e # for ios
```
Now you should see the RNTester app being open, and the defined test being run.
## Adding new tests (and project structure)
This project has 2 main folders:
- `apps`, where, as you have seen above, the iOS/Android RNTester apps need to be put so that appium will pick them and install in the emulator/simulator consistently.
- `tests`, where the tests and referencing files all live. The substructure is as follows:
- `screens` -> in this folder, you will find `*.screen.js` files, where each file represents a navigation screen for RNTester. So there are 3 root ones (`apis`, `bookmarks`, `components`) and then for subscreens, there's a folder with the same name - currently, that's only `components` that contains `buttonComponent.screen.js`. The content of these files is what was earlier mentioned as "references": they provide an easy way to define all elements present in said screen, so that they can be used for tests.
- `specs` -> this folder follows a similar 1:1 mapping to the RNTester screens, but for the tests: for each screen (or subscreen) there's a dedicated `*.test.js` file (such as `buttonComponentScreen.test.js`). Ideally, in this file the Jest tests are standard, leveraging the `*.screen.js` counterpart for the details of defining how Appium/WDIO can reach those elements on screen.
When adding a new test, please ensure that you follow this pattern and add the relevant test in the right screen file / screen test file. Use the files mentioned above as examples.
+1
View File
@@ -0,0 +1 @@
Put the *.app and *.apk files here.
+14
View File
@@ -0,0 +1,14 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
module.exports = {
presets: [['@babel/preset-env', {targets: {node: 'current'}}]],
plugins: ['@babel/plugin-transform-flow-strip-types'],
};
+61
View File
@@ -0,0 +1,61 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
const path = require('path');
type Capabilities = {
platformName: 'Android' | 'iOS',
'appium:platformVersion': string,
'appium:deviceName': string,
'appium:app': string,
'appium:automationName': 'UiAutomator2' | 'XCUITest',
'appium:newCommandTimeout'?: number,
};
let capabilities: Capabilities;
const android = {
platformName: 'Android',
'appium:platformVersion': '13.0',
'appium:deviceName': 'Android Emulator',
'appium:app': path.join(process.cwd(), '/apps/rn-tester.apk'),
'appium:automationName': 'UiAutomator2',
'appium:newCommandTimeout': 240,
};
const ios = {
platformName: 'iOS',
'appium:platformVersion': '16.4',
'appium:deviceName': 'iPhone 14',
'appium:automationName': 'XCUITest',
'appium:app': path.join(process.cwd(), '/apps/rn-tester.app'),
};
// check that E2E_DEVICE exists, is a string and its either "ios" or "android"
if (!process.env.E2E_DEVICE) {
throw new Error('E2E_DEVICE environment variable is not defined');
} else if (typeof process.env.E2E_DEVICE !== 'string') {
throw new Error('E2E_DEVICE environment variable is not a string');
} else if (
process.env.E2E_DEVICE !== 'ios' &&
process.env.E2E_DEVICE !== 'android'
) {
throw new Error('E2E_DEVICE environment variable is not "ios" or "android"');
}
if (process.env.E2E_DEVICE === 'android') {
capabilities = android;
}
if (process.env.E2E_DEVICE === 'ios') {
capabilities = ios;
}
export default capabilities;
+17
View File
@@ -0,0 +1,17 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
module.exports = {
testTimeout: 120000,
bail: 0,
setupFilesAfterEnv: ['./jest.setup.js'],
testMatch: ['**/specs/**/*.js'],
maxWorkers: 1,
};
+38
View File
@@ -0,0 +1,38 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
const wdio = require('webdriverio');
import capabilities from './e2e-config.js';
import {beforeEach, afterEach, jest} from '@jest/globals';
jest.retryTimes(3);
let driver: any;
const config = {
path: '/wd/hub',
host: 'localhost',
port: 4723,
waitforTimeout: 60000,
logLevel: 'error',
capabilities: {
...capabilities,
},
};
beforeEach(async () => {
driver = await wdio.remote(config);
});
afterEach(async () => {
console.info('[afterAll] Done with testing!');
await driver.deleteSession();
});
export {driver};
+30
View File
@@ -0,0 +1,30 @@
{
"name": "@react-native/tester-e2e",
"private": true,
"version": "0.0.1",
"license": "MIT",
"description": "React Native E2E tester app.",
"homepage": "https://github.com/facebook/react-native/tree/HEAD/packages/rn-tester-e2e",
"repository": {
"type": "git",
"url": "git@github.com:facebook/react-native.git",
"directory": "packages/rn-tester-e2e"
},
"scripts": {
"test": "jest --runInBand",
"test-android-e2e": "E2E_DEVICE=\"android\" yarn test",
"test-ios-e2e": "E2E_DEVICE=\"ios\" yarn test"
},
"devDependencies": {
"appium": "^2.0.0",
"appium-uiautomator2-driver": "^2.29.1",
"appium-xcuitest-driver": "^4.32.10",
"eslint": "^8.19.0",
"jest": "^29.2.1",
"webdriverio": "^7.32.0",
"@babel/core": "^7.20.0",
"@babel/preset-env": "^7.20.0",
"@babel/plugin-transform-flow-strip-types": "^7.20.0",
"@types/jest": "^29.2.1"
}
}
@@ -0,0 +1,52 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
import {driver} from '../../jest.setup';
type PlatformsReference = {
ios: string,
android: string,
};
class Utils {
async checkElementExistence(locator: string): Promise<boolean> {
await driver.$(locator).waitForDisplayed();
return driver.$(locator).isDisplayed();
}
async clickElement(locator: string): Promise<void> {
await driver.$(locator).waitForDisplayed();
await driver.$(locator).click();
}
async getElementText(locator: string): Promise<string> {
await driver.$(locator).waitForDisplayed();
return driver.$(locator).getText();
}
platformSelect(platforms: PlatformsReference): string {
// if something goes wrong, we fallback to ios. But it should never happent, the process will fail way earlier.
return platforms[process?.env?.E2E_DEVICE || 'ios'];
}
}
export const iOSLabel = (label: string): string => {
return `[label="${label}"]`;
};
export const androidWidget = (
type: string,
selector: string,
id: string,
): string => {
return `//android.widget.${type}[@${selector}="${id}"]`;
};
export const UtilsSingleton: Utils = new Utils();
@@ -0,0 +1,15 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
// root level screen in RNTester: APIs
class ApisScreen {}
export const ApisScreenSingleton: ApisScreen = new ApisScreen();
@@ -0,0 +1,15 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
// root level screen in RNTester: Bookmarks
class BookmarksScreen {}
export const BookmarksScreenSingleton: BookmarksScreen = new BookmarksScreen();
@@ -0,0 +1,40 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
import {UtilsSingleton as Utils, iOSLabel} from '../helpers/utils';
// root level screen in RNTester: Components
const buttonComponentLabel = 'Button Simple React Native button component.';
type ComponentsScreenType = {
buttonComponentLabelElement: string,
checkButtonComponentIsDisplayed: () => Promise<boolean>,
clickButtonComponent: () => Promise<void>,
};
export const ComponentsScreen: ComponentsScreenType = {
// Reference in the top level Component list
buttonComponentLabelElement: Utils.platformSelect({
ios: iOSLabel(buttonComponentLabel),
android: `~${buttonComponentLabel}`,
}),
// Methods to interact with top level elements in the list
checkButtonComponentIsDisplayed: async function (
this: ComponentsScreenType,
): Promise<boolean> {
return await Utils.checkElementExistence(this.buttonComponentLabelElement);
},
clickButtonComponent: async function (
this: ComponentsScreenType,
): Promise<void> {
await Utils.clickElement(this.buttonComponentLabelElement);
},
};
@@ -0,0 +1,95 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
import {
UtilsSingleton as Utils,
iOSLabel,
androidWidget,
} from '../../helpers/utils';
type ButtonComponentScreenType = {
buttonScreenElement: string,
btnSubmitElement: string,
inputSearchElement: string,
btnOKElement: string,
btnCancelElement: string,
submitAlertBoxElement: string,
cancelAlertBoxElement: string,
checkButtonsScreenIsDisplayed: () => Promise<string>,
clickSubmitApplication: () => Promise<void>,
clickCancelApplication: () => Promise<void>,
getCancelAlertText: () => Promise<string>,
getSubmitAlertText: () => Promise<string>,
clickOkButton: () => Promise<void>,
};
export const ButtonComponentScreen: ButtonComponentScreenType = {
// reference in the Components list
buttonScreenElement: Utils.platformSelect({
ios: iOSLabel('Button'),
android: androidWidget('ViewGroup', 'text', 'Button'),
}),
// References to elements within the Button Component screen
btnSubmitElement: Utils.platformSelect({
ios: iOSLabel('Press to submit your application!'),
android: androidWidget('Button', 'resource-id', 'button_default_styling'),
}),
inputSearchElement: Utils.platformSelect({
ios: iOSLabel('example_search'),
android: androidWidget('EditText', 'resource-id', 'example_search'),
}),
btnOKElement: Utils.platformSelect({
ios: iOSLabel('OK'),
android: androidWidget('Button', 'text', 'OK'),
}),
btnCancelElement: Utils.platformSelect({
ios: iOSLabel('Press to cancel your application!'),
android: androidWidget('Button', 'resource-id', 'cancel_button'),
}),
submitAlertBoxElement: Utils.platformSelect({
ios: iOSLabel('Your application has been submitted!'),
android: androidWidget('TextView', 'resource-id', 'android:id/alertTitle'),
}),
cancelAlertBoxElement: Utils.platformSelect({
ios: iOSLabel('Your application has been cancelled!'),
android: androidWidget('TextView', 'resource-id', 'android:id/alertTitle'),
}),
// Methods to interact with the elements
checkButtonsScreenIsDisplayed: async function (
this: ButtonComponentScreenType,
): Promise<string> {
return await Utils.getElementText(this.buttonScreenElement);
},
clickSubmitApplication: async function (
this: ButtonComponentScreenType,
): Promise<void> {
await Utils.clickElement(this.btnSubmitElement);
},
clickCancelApplication: async function (
this: ButtonComponentScreenType,
): Promise<void> {
await Utils.clickElement(this.btnCancelElement);
},
getCancelAlertText: async function (
this: ButtonComponentScreenType,
): Promise<string> {
return await Utils.getElementText(this.cancelAlertBoxElement);
},
getSubmitAlertText: async function (
this: ButtonComponentScreenType,
): Promise<string> {
return await Utils.getElementText(this.submitAlertBoxElement);
},
clickOkButton: async function (
this: ButtonComponentScreenType,
): Promise<void> {
await Utils.clickElement(this.btnOKElement);
},
};
@@ -0,0 +1,46 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @flow
* @format
*/
const {ComponentsScreen} = require('../../screens/components.screen.js');
const {
ButtonComponentScreen,
} = require('../../screens/components/buttonComponent.screen.js');
// fixed variables
const submitText = 'Your application has been submitted!';
const cancelText = 'Your application has been cancelled!';
describe('Test is checking submit button', () => {
test('Should view properly submit alert text', async () => {
expect(
await ComponentsScreen.checkButtonComponentIsDisplayed(),
).toBeTruthy();
await ComponentsScreen.clickButtonComponent();
await ButtonComponentScreen.clickSubmitApplication();
expect(await ButtonComponentScreen.getSubmitAlertText()).toContain(
submitText,
);
await ButtonComponentScreen.clickOkButton();
});
});
describe('Test is checking cancel button', () => {
test('Should view properly submit cancel text', async () => {
expect(
await ComponentsScreen.checkButtonComponentIsDisplayed(),
).toBeTruthy();
await ComponentsScreen.clickButtonComponent();
await ButtonComponentScreen.clickCancelApplication();
expect(await ButtonComponentScreen.getCancelAlertText()).toContain(
cancelText,
);
await ButtonComponentScreen.clickOkButton();
});
});
+3236 -896
View File
File diff suppressed because it is too large Load Diff