diff --git a/.github/actions/maestro-android/action.yml b/.github/actions/maestro-android/action.yml index 58a7351da64..7f760573c94 100644 --- a/.github/actions/maestro-android/action.yml +++ b/.github/actions/maestro-android/action.yml @@ -38,6 +38,10 @@ inputs: required: false default: /tmp/maestro-android-state/results.json description: The path used to persist per-flow test results between retries + exclude-tags: + required: false + default: '' + description: Comma-separated flow tags to exclude from the test run runs: using: composite @@ -77,7 +81,7 @@ runs: cores: '4' disable-animations: false avd-name: e2e_emulator - script: node .github/workflow-scripts/maestro-android.js ${{ inputs.app-path }} ${{ inputs.app-id }} ${{ inputs.maestro-flow }} ${{ inputs.flavor }} ${{ inputs.working-directory }} ${{ inputs.test-state-path }} + script: node .github/workflow-scripts/maestro-android.js ${{ inputs.app-path }} ${{ inputs.app-id }} ${{ inputs.maestro-flow }} ${{ inputs.flavor }} ${{ inputs.working-directory }} ${{ inputs.test-state-path }} "${{ inputs.exclude-tags }}" - name: Normalize APP_ID id: normalize-app-id shell: bash diff --git a/.github/workflow-scripts/__tests__/maestro-android-test.js b/.github/workflow-scripts/__tests__/maestro-android-test.js index 77b5711030b..c07def601b2 100644 --- a/.github/workflow-scripts/__tests__/maestro-android-test.js +++ b/.github/workflow-scripts/__tests__/maestro-android-test.js @@ -14,6 +14,7 @@ const path = require('path'); const { collectFlows, executeFlowSuite, + filterFlowsByTags, loadState, } = require('../maestro-android'); @@ -45,6 +46,26 @@ describe('Maestro Android runner', () => { ]); }); + it('excludes flows with matching tags', () => { + const releaseOnlyFlow = path.join( + temporaryDirectory, + 'android-release-only.yml', + ); + const regularFlow = path.join(temporaryDirectory, 'regular.yml'); + fs.writeFileSync( + releaseOnlyFlow, + 'appId: x\ntags:\n - android-release-only\n---\n- launchApp\n', + ); + fs.writeFileSync(regularFlow, 'appId: x\n---\n- launchApp\n'); + + expect( + filterFlowsByTags( + [releaseOnlyFlow, regularFlow], + ['android-release-only'], + ), + ).toEqual([regularFlow]); + }); + it('runs every flow and retries only flows that have not passed', () => { const flows = ['first.yml', 'second.yml', 'third.yml'].map(file => path.join(temporaryDirectory, file), diff --git a/.github/workflow-scripts/maestro-android.js b/.github/workflow-scripts/maestro-android.js index 7b2365512ef..d409afb42a5 100644 --- a/.github/workflow-scripts/maestro-android.js +++ b/.github/workflow-scripts/maestro-android.js @@ -13,7 +13,7 @@ const path = require('path'); const usage = ` === Usage === -node maestro-android.js [test_state_path] +node maestro-android.js [test_state_path] [exclude_tags] @param {string} appPath - Path to the app APK @param {string} appId - App ID that needs to be launched @@ -21,6 +21,7 @@ node maestro-android.js { + const tags = getFlowTags(flow); + const excludedTag = tags.find(tag => excludeTags.includes(tag)); + if (excludedTag != null) { + console.info(`Skipping flow tagged ${excludedTag}: ${flow}`); + return false; + } + return true; + }); +} + function getFlowKey(flow) { return path .relative(process.cwd(), path.resolve(flow)) @@ -274,7 +308,7 @@ function executeFlowSuite({ } async function main(args = process.argv.slice(2)) { - if (args.length < 5 || args.length > 6) { + if (args.length < 5 || args.length > 7) { throw new Error(`Invalid number of arguments.\n${usage}`); } @@ -284,6 +318,10 @@ async function main(args = process.argv.slice(2)) { const isDebug = args[3] === 'debug'; const workingDirectory = args[4]; const statePath = args[5] ?? DEFAULT_STATE_PATH; + const excludeTags = (args[6] ?? '') + .split(',') + .map(tag => tag.trim()) + .filter(Boolean); console.info('\n=============================='); console.info('Running tests for Android with the following parameters:'); @@ -293,6 +331,7 @@ async function main(args = process.argv.slice(2)) { console.info(`IS_DEBUG: ${isDebug}`); console.info(`WORKING_DIRECTORY: ${workingDirectory}`); console.info(`TEST_STATE_PATH: ${statePath}`); + console.info(`EXCLUDE_TAGS: ${excludeTags.join(',') || ''}`); console.info('==============================\n'); logAndroidAbiConfiguration(); @@ -338,7 +377,7 @@ async function main(args = process.argv.slice(2)) { let error = null; try { - const flows = collectFlows(maestroFlow); + const flows = filterFlowsByTags(collectFlows(maestroFlow), excludeTags); const state = loadState(statePath); console.info(`Start testing ${flows.length} flow(s)`); executeFlowSuite({flows, appId, state, statePath}); @@ -378,6 +417,7 @@ if (require.main === module) { module.exports = { collectFlows, executeFlowSuite, + filterFlowsByTags, formatResults, loadState, }; diff --git a/.github/workflows/e2e-android-rntester.yml b/.github/workflows/e2e-android-rntester.yml index dca0ae68ced..7b98d00fed2 100644 --- a/.github/workflows/e2e-android-rntester.yml +++ b/.github/workflows/e2e-android-rntester.yml @@ -71,6 +71,10 @@ jobs: emulator-api-level: '35' emulator-target: google_apis emulator-arch: x86_64 + # These timing-sensitive flows remain covered by the release APK. + # Running the debug ARM64 APK through native translation can delay + # React commits long enough for Maestro to miss taps and assertions. + exclude-tags: ${{ matrix.flavor == 'debug' && 'android-release-only' || '' }} - name: Store per-flow test state if: always() uses: actions/upload-artifact@v6 diff --git a/packages/rn-tester/.maestro/flatlist-append-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-append-maintainvisible.yml index 8698e0ecd95..47a3097fe68 100644 --- a/packages/rn-tester/.maestro/flatlist-append-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-append-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition with append (baseline) # Appending items should NOT affect scroll offset (delta ~0) appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-complex-mutations-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-complex-mutations-maintainvisible.yml index 701074f2122..72781f1a9a3 100644 --- a/packages/rn-tester/.maestro/flatlist-complex-mutations-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-complex-mutations-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition — complex concurrent mutations # Tests prepend + append + delete in sequence appId: ${APP_ID} +tags: + - android-release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-delete-anchor-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-delete-anchor-maintainvisible.yml index 498c919bc0d..7e6dd3a5f6c 100644 --- a/packages/rn-tester/.maestro/flatlist-delete-anchor-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-delete-anchor-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition — delete anchor item # When the anchor item (first visible) is deleted, MVCP should select a new anchor appId: ${APP_ID} +tags: + - android-release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-delete-middle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-delete-middle-maintainvisible.yml index c807d1770e9..8ca036af020 100644 --- a/packages/rn-tester/.maestro/flatlist-delete-middle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-delete-middle-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition — delete from middle # When items are deleted from the middle, MVCP should adjust scroll offset appId: ${APP_ID} +tags: + - android-release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-empty-list-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-empty-list-maintainvisible.yml index 7dee4a0baaf..be79c35b7a6 100644 --- a/packages/rn-tester/.maestro/flatlist-empty-list-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-empty-list-maintainvisible.yml @@ -3,6 +3,8 @@ # _firstVisibleView.frame on nil returns {0,0}, causing incorrect scroll # correction. appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-first-prepend-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-first-prepend-maintainvisible.yml index 1b90a7454a7..8305cbc874c 100644 --- a/packages/rn-tester/.maestro/flatlist-first-prepend-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-first-prepend-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition — first prepend only # Single prepend with fixed-height items: delta should be ~44px (40px height + 4px margin) appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-add50-reset-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-add50-reset-maintainvisible.yml index 79a805e89c2..04d9c3c2072 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-add50-reset-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-add50-reset-maintainvisible.yml @@ -2,6 +2,8 @@ # Verifies that after horizontal mode with 50 prepended items and scroll to 500, # reset returns offset to 0. appId: ${APP_ID} +tags: + - android-release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-inverted-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-inverted-maintainvisible.yml index c9b60ab70c2..c53fcd8db23 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-inverted-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-inverted-maintainvisible.yml @@ -2,6 +2,8 @@ # Items are 200px wide + 4px margin = 204px each # In inverted mode, prepending adds items to the right end appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-inverted-recycle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-inverted-recycle-maintainvisible.yml index b86ebefe8b7..ce9234cae13 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-inverted-recycle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-inverted-recycle-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition with view recycling in horizontal + inverted mode # Items are 200px wide + 4px margin = 204px each appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-maintainvisible.yml index 4289802fc0b..8258d117230 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition in horizontal mode # Horizontal: items are 200px wide, delta should be ~204px (200px + 4px margin) appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-recycle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-recycle-maintainvisible.yml index 5267afec16e..c63531017e3 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-recycle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-recycle-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition with view recycling in horizontal mode # Items are 200px wide + 4px margin = 204px each appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-inverted-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-inverted-maintainvisible.yml index 1efb844b3b1..28f585f95f4 100644 --- a/packages/rn-tester/.maestro/flatlist-inverted-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-inverted-maintainvisible.yml @@ -2,6 +2,8 @@ # Delta is +44 (same as non-inverted) — frame-based delta measures actual frame shift, # not logical order. Prepending shifts anchor view frame downward in both modes. appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-inverted-recycle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-inverted-recycle-maintainvisible.yml index 928e968c7c6..ded9fe4c987 100644 --- a/packages/rn-tester/.maestro/flatlist-inverted-recycle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-inverted-recycle-maintainvisible.yml @@ -3,6 +3,8 @@ # With windowSize=3, only ~3 pages of items are rendered # In inverted mode, items display in reverse order but MVCP delta behavior is the same appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-maintainvisible.yml index ff86df5d49b..d0e05fc6287 100644 --- a/packages/rn-tester/.maestro/flatlist-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-maintainvisible.yml @@ -7,6 +7,8 @@ # - After prepend: record offset # - Delta should equal height of prepended items (~44px per item with margin) appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait @@ -87,6 +89,8 @@ appId: ${APP_ID} --- # Test that user scroll is not interrupted during prepend appId: ${APP_ID} +tags: + - android-release-only --- - launchApp - assertVisible: 'Components' diff --git a/packages/rn-tester/.maestro/flatlist-momentum-scroll-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-momentum-scroll-maintainvisible.yml index dbf2fe96b2a..820c6586db2 100644 --- a/packages/rn-tester/.maestro/flatlist-momentum-scroll-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-momentum-scroll-maintainvisible.yml @@ -2,6 +2,8 @@ # Verifies that scroll position remains stable after momentum scroll completes # post-prepend. MVCP correction runs asynchronously in didMountItems. appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-orientation-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-orientation-maintainvisible.yml index 003946fa209..8ecefd77df5 100644 --- a/packages/rn-tester/.maestro/flatlist-orientation-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-orientation-maintainvisible.yml @@ -6,6 +6,8 @@ # This test verifies that MVCP survives an orientation change and continues # to work correctly after the change. appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-prepend-delete-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-prepend-delete-maintainvisible.yml index f0a27be47b0..ad116f7a61e 100644 --- a/packages/rn-tester/.maestro/flatlist-prepend-delete-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-prepend-delete-maintainvisible.yml @@ -4,6 +4,8 @@ # The native side is unaffected by bottom deletes since MVCP only looks at # the first visible view, but re-ordering edge case is untested. appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-pull-to-refresh-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-pull-to-refresh-maintainvisible.yml index 2fe197270ae..31445c02d9a 100644 --- a/packages/rn-tester/.maestro/flatlist-pull-to-refresh-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-pull-to-refresh-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition — pull-to-refresh pattern # Simulates scroll-to-top then prepend (like pull-to-refresh with new items) appId: ${APP_ID} +tags: + - android-release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-rapid-prepends-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-rapid-prepends-maintainvisible.yml index 0f126d5f664..c7434fb6a12 100644 --- a/packages/rn-tester/.maestro/flatlist-rapid-prepends-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-rapid-prepends-maintainvisible.yml @@ -2,6 +2,8 @@ # Exercises the throttle edge case where pendingScrollUpdateCount may not decrement # promptly, blocking render window updates. All prepends fired without waiting. appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-recycle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-recycle-maintainvisible.yml index 0706607843b..16cb6cf3132 100644 --- a/packages/rn-tester/.maestro/flatlist-recycle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-recycle-maintainvisible.yml @@ -2,6 +2,8 @@ # Items are 40px tall + 4px margin = 44px each # With windowSize=3, only ~3 pages of items are rendered appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-scrolltooffset-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-scrolltooffset-maintainvisible.yml index 8a5b077c570..db298cc2b2f 100644 --- a/packages/rn-tester/.maestro/flatlist-scrolltooffset-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-scrolltooffset-maintainvisible.yml @@ -3,6 +3,8 @@ # during MVCP active causes additive correction (MVCP delta added on top # of the scrollTo target). appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-throttle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-throttle-maintainvisible.yml index 945a8032a09..5b0e4103f5e 100644 --- a/packages/rn-tester/.maestro/flatlist-throttle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-throttle-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition with scroll event throttle # Throttle affects timing but not final delta: should be ~44px appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-variable-height-first-prepend-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-variable-height-first-prepend-maintainvisible.yml index 4409227b927..385b1d20952 100644 --- a/packages/rn-tester/.maestro/flatlist-variable-height-first-prepend-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-variable-height-first-prepend-maintainvisible.yml @@ -1,6 +1,8 @@ # Test variable-height items with first prepend # Single prepend with variable height: delta should be 28-112px appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-variable-height-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-variable-height-maintainvisible.yml index 0f5580e0909..da034ec4b2b 100644 --- a/packages/rn-tester/.maestro/flatlist-variable-height-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-variable-height-maintainvisible.yml @@ -1,6 +1,8 @@ # Test FlatList maintainVisibleContentPosition with variable-height items # Delta should be between 28-112px (random height from [30,50,70,90,110]) appId: ${APP_ID} +tags: + - android-release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/image-wide-gamut.yml b/packages/rn-tester/.maestro/image-wide-gamut.yml index f607bc87542..5d7ca283527 100644 --- a/packages/rn-tester/.maestro/image-wide-gamut.yml +++ b/packages/rn-tester/.maestro/image-wide-gamut.yml @@ -3,8 +3,8 @@ tags: - android-only --- # Android-only: alpha transparency + sRGB vs Display-P3 wide-gamut. Color -# fidelity needs a screenshot, so we assert load status only. Flaky on the iOS -# E2E job, so guarded to Android; iOS skips this flow (empty=pass). +# fidelity is covered by a screenshot captured from the ARM64 APK on the API 35 +# sRGB emulator. The flow is guarded because it is flaky on the iOS E2E job. - runFlow: when: platform: Android diff --git a/packages/rn-tester/.maestro/screenshots/image-wide-gamut-android.png b/packages/rn-tester/.maestro/screenshots/image-wide-gamut-android.png index 0bf56881588..3b7ca2ca47c 100644 Binary files a/packages/rn-tester/.maestro/screenshots/image-wide-gamut-android.png and b/packages/rn-tester/.maestro/screenshots/image-wide-gamut-android.png differ