Use BrowserStack to run macOS CI tests (#7836)

* Use BrowserStack to run macOS CI tests
* Add more browserstack script targets and skip safari on assets blocking requests or failing with software aes
* Pin browserstack/github-actions to v1.0.4 (93aebce225b754566349151c0676b26b005e591b)
This commit is contained in:
Rob Walch
2026-05-08 13:05:55 -07:00
committed by GitHub
parent 8f93bebde2
commit 209d3a06f6
5 changed files with 251 additions and 30 deletions
+42 -19
View File
@@ -20,6 +20,7 @@ jobs:
cancel-in-progress: true
outputs:
canUseSauce: ${{ steps.check_sauce_access.outputs.result == 'true' }}
canUseBrowserStack: ${{ steps.check_browserstack_access.outputs.result == 'true' }}
tag: ${{ steps.extract_tag.outputs.result }}
isMainBranch: ${{ github.ref == 'refs/heads/master' }}
steps:
@@ -33,6 +34,16 @@ jobs:
CI: true
SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
- name: check browserstack access
id: check_browserstack_access
run: |
if ! [[ -z "$BROWSERSTACK_USERNAME" ]] && ! [[ -z "$BROWSERSTACK_ACCESS_KEY" ]]; then
echo "result=true" >> $GITHUB_OUTPUT
fi
env:
CI: true
BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
- name: extract tag
id: extract_tag
uses: actions/github-script@v8
@@ -406,7 +417,8 @@ jobs:
OS: ${{ matrix.os }}
test_functional_optional:
needs: test_functional_required
needs: [config, test_functional_required]
if: needs.config.outputs.canUseBrowserStack == 'true'
runs-on: ubuntu-latest
concurrency:
group: 'build:test_functional_optional:${{ matrix.config }}:${{ github.ref }}'
@@ -415,19 +427,18 @@ jobs:
name: test_functional_optional (${{ matrix.config }})
strategy:
fail-fast: false
max-parallel: 8
max-parallel: 4
matrix:
include:
- config: safari-macOS_13
- config: safari-macOS_15
ua: safari
os: macOS 13
- config: firefox-win_10
ua: firefox
os: Windows 10
- config: chrome-mac_13-79.0
os: macOS 15
- config: chrome-macOS_14
ua: chrome
os: macOS 12
uaVersion: '79.0'
os: macOS 14
- config: firefox-macOS_15
ua: firefox
os: macOS 15
steps:
- uses: actions/checkout@v5
@@ -454,13 +465,17 @@ jobs:
with:
name: build
- name: start SauceConnect tunnel
uses: saucelabs/sauce-connect-action@0ca0e6ce3a5513d6bec2a54044a536c3da3a53fb # v2
- name: setup BrowserStack env
uses: browserstack/github-actions/setup-env@93aebce225b754566349151c0676b26b005e591b # v1.0.4
with:
username: ${{ secrets.SAUCE_USERNAME }}
accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
tunnelIdentifier: ${{ github.run_id }}-${{ matrix.config }}
retryTimeout: 300
username: ${{ secrets.BROWSERSTACK_USERNAME }}
access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
- name: start BrowserStack Local
uses: browserstack/github-actions/setup-local@93aebce225b754566349151c0676b26b005e591b # v1.0.4
with:
local-testing: start
local-identifier: ${{ github.run_id }}-${{ matrix.config }}
- name: install
run: |
@@ -473,9 +488,17 @@ jobs:
npm run test:func
env:
CI: true
SAUCE_TUNNEL_ID: ${{ github.run_id }}-${{ matrix.config }}
SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
BROWSERSTACK: 1
BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
BROWSERSTACK_LOCAL_IDENTIFIER: ${{ github.run_id }}-${{ matrix.config }}
BROWSERSTACK_BUILD_ID: ${{ github.run_id }}-${{ matrix.config }}
UA: ${{ matrix.ua }}
UA_VERSION: ${{ matrix.uaVersion }}
OS: ${{ matrix.os }}
- name: stop BrowserStack Local
if: always()
uses: browserstack/github-actions/setup-local@93aebce225b754566349151c0676b26b005e591b # v1.0.4
with:
local-testing: stop
+55
View File
@@ -34,6 +34,7 @@
"@typescript-eslint/parser": "8.54.0",
"babel-loader": "10.1.1",
"babel-plugin-transform-remove-console": "6.9.4",
"browserstack-local": "1.5.13",
"chai": "6.2.2",
"chart.js": "2.9.4",
"chromedriver": "146.0.6",
@@ -5448,6 +5449,19 @@
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
}
},
"node_modules/browserstack-local": {
"version": "1.5.13",
"resolved": "https://registry.npmjs.org/browserstack-local/-/browserstack-local-1.5.13.tgz",
"integrity": "sha512-7helY+Ms3ss4BtIQZTIyshdAFZSvS9A7ZpEB9stRaobeZ9BM1BkJFTuMakQNTOj78llv0+/qDI5Ak+bkGWV1xg==",
"dev": true,
"license": "MIT",
"dependencies": {
"agent-base": "^6.0.2",
"https-proxy-agent": "^5.0.1",
"is-running": "^2.1.0",
"tree-kill": "^1.2.2"
}
},
"node_modules/buffer-crc32": {
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@@ -8874,6 +8888,13 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/is-running": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-running/-/is-running-2.1.0.tgz",
"integrity": "sha512-mjJd3PujZMl7j+D395WTIO5tU5RIDBfVSRtRR4VOJou3H66E38UjbjvDGh3slJzPuolsb+yQFqwHNNdyp5jg3w==",
"dev": true,
"license": "BSD"
},
"node_modules/is-set": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.3.tgz",
@@ -13273,6 +13294,16 @@
"node": ">=0.6"
}
},
"node_modules/tree-kill": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
"integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==",
"dev": true,
"license": "MIT",
"bin": {
"tree-kill": "cli.js"
}
},
"node_modules/trough": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/trough/-/trough-1.0.5.tgz",
@@ -18111,6 +18142,18 @@
"update-browserslist-db": "^1.2.0"
}
},
"browserstack-local": {
"version": "1.5.13",
"resolved": "https://registry.npmjs.org/browserstack-local/-/browserstack-local-1.5.13.tgz",
"integrity": "sha512-7helY+Ms3ss4BtIQZTIyshdAFZSvS9A7ZpEB9stRaobeZ9BM1BkJFTuMakQNTOj78llv0+/qDI5Ak+bkGWV1xg==",
"dev": true,
"requires": {
"agent-base": "^6.0.2",
"https-proxy-agent": "^5.0.1",
"is-running": "^2.1.0",
"tree-kill": "^1.2.2"
}
},
"buffer-crc32": {
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@@ -20570,6 +20613,12 @@
"hasown": "^2.0.2"
}
},
"is-running": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-running/-/is-running-2.1.0.tgz",
"integrity": "sha512-mjJd3PujZMl7j+D395WTIO5tU5RIDBfVSRtRR4VOJou3H66E38UjbjvDGh3slJzPuolsb+yQFqwHNNdyp5jg3w==",
"dev": true
},
"is-set": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.3.tgz",
@@ -23787,6 +23836,12 @@
"integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
"dev": true
},
"tree-kill": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
"integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==",
"dev": true
},
"trough": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/trough/-/trough-1.0.5.tgz",
+6
View File
@@ -60,6 +60,11 @@
"test:func": "BABEL_ENV=development mocha --require @babel/register tests/functional/auto/setup.js --timeout 40000 --exit",
"test:func:light": "BABEL_ENV=development HLSJS_LIGHT=1 mocha --require @babel/register tests/functional/auto/setup.js --timeout 40000 --exit",
"test:func:sauce": "SAUCE=1 UA=safari OS='OS X 10.15' BABEL_ENV=development mocha --require @babel/register tests/functional/auto/setup.js --timeout 40000 --exit",
"test:func:browserstack": "BROWSERSTACK=1 UA=safari OS='macOS 15' BABEL_ENV=development mocha --require @babel/register tests/functional/auto/setup.js --timeout 40000 --exit",
"test:func:android": "BROWSERSTACK=1 UA=chrome DEVICE='Samsung Galaxy S24' OS='Android 14' BABEL_ENV=development mocha --require @babel/register tests/functional/auto/setup.js --timeout 40000 --exit",
"test:func:android:tablet": "BROWSERSTACK=1 UA=chrome DEVICE='Samsung Galaxy Tab S9' OS='Android 13' BABEL_ENV=development mocha --require @babel/register tests/functional/auto/setup.js --timeout 40000 --exit",
"test:func:ios": "BROWSERSTACK=1 UA=safari DEVICE='iPhone 16' OS='iOS 18' BABEL_ENV=development mocha --require @babel/register tests/functional/auto/setup.js --timeout 40000 --exit",
"test:func:ipad": "BROWSERSTACK=1 UA=safari DEVICE='iPad Pro 12.9 2022' OS='iOS 16' BABEL_ENV=development mocha --require @babel/register tests/functional/auto/setup.js --timeout 40000 --exit",
"type-check": "tsc --noEmit",
"type-check:watch": "npm run type-check -- --watch",
"prepare": "husky"
@@ -92,6 +97,7 @@
"@typescript-eslint/parser": "8.54.0",
"babel-loader": "10.1.1",
"babel-plugin-transform-remove-console": "6.9.4",
"browserstack-local": "1.5.13",
"chai": "6.2.2",
"chart.js": "2.9.4",
"chromedriver": "146.0.6",
+146 -11
View File
@@ -8,7 +8,10 @@ const until = webdriver.until;
require('chromedriver');
const HttpServer = require('http-server');
const streams = require('../../test-streams');
const useSauce = !!process.env.SAUCE || !!process.env.SAUCE_TUNNEL_ID;
const useBrowserStack = !!process.env.BROWSERSTACK;
const useSauce =
!useBrowserStack && (!!process.env.SAUCE || !!process.env.SAUCE_TUNNEL_ID);
const useRemote = useSauce || useBrowserStack;
const HlsjsLightBuild = !!process.env.HLSJS_LIGHT;
const { expect } = require('chai');
@@ -31,7 +34,27 @@ let browser;
const printDebugLogs = false;
// Setup browser config data from env vars
if (useSauce) {
if (useBrowserStack) {
const UA = process.env.UA;
if (!UA) {
throw new Error('No test browser name.');
}
const OS = process.env.OS;
if (!OS) {
throw new Error('No test browser platform.');
}
if (
!process.env.BROWSERSTACK_USERNAME ||
!process.env.BROWSERSTACK_ACCESS_KEY
) {
throw new Error('Missing BrowserStack auth.');
}
browserConfig.name = UA;
browserConfig.platform = OS;
} else if (useSauce) {
const UA = process.env.UA;
if (!UA) {
throw new Error('No test browser name.');
@@ -61,12 +84,12 @@ if (browserConfig.platform) {
}
// Launch static server
if (useSauce || !HLSJS_TEST_BASE) {
if (useRemote || !HLSJS_TEST_BASE) {
HttpServer.createServer({
showDir: false,
autoIndex: false,
root: './',
}).listen(8000, useSauce ? '0.0.0.0' : '127.0.0.1');
}).listen(8000, useRemote ? '0.0.0.0' : '127.0.0.1');
}
const wait = (ms) => new Promise((resolve) => global.setTimeout(resolve, ms));
@@ -536,10 +559,58 @@ async function sauceDisconnect() {
});
}
function mapOSToBrowserStack(os) {
const mapping = {
'macOS 12': { os: 'OS X', osVersion: 'Monterey' },
'macOS 13': { os: 'OS X', osVersion: 'Ventura' },
'macOS 14': { os: 'OS X', osVersion: 'Sonoma' },
'macOS 15': { os: 'OS X', osVersion: 'Sequoia' },
'Windows 10': { os: 'Windows', osVersion: '10' },
'Windows 11': { os: 'Windows', osVersion: '11' },
};
return mapping[os] || { os: os, osVersion: '' };
}
let browserstackLocalInstance;
async function startBrowserStackLocal(localIdentifier) {
const browserStackLocal = require('browserstack-local');
return new Promise(function (resolve, reject) {
console.log(`Starting BrowserStack Local. Identifier: ${localIdentifier}`);
browserstackLocalInstance = new browserStackLocal.Local();
const opts = {
key: process.env.BROWSERSTACK_ACCESS_KEY,
localIdentifier: localIdentifier,
forceLocal: true,
};
browserstackLocalInstance.start(opts, function (err) {
if (err) {
console.error('BrowserStack Local error:', err.message);
reject(err);
return;
}
console.log('BrowserStack Local connected');
resolve(browserstackLocalInstance);
});
});
}
async function stopBrowserStackLocal() {
return new Promise(function (resolve) {
if (!browserstackLocalInstance) {
resolve();
return;
}
browserstackLocalInstance.stop(function () {
console.log('BrowserStack Local disconnected');
resolve();
});
});
}
function getPageURLComponents() {
return {
base:
useSauce || !HLSJS_TEST_BASE ? 'http://localhost:8000' : HLSJS_TEST_BASE,
useRemote || !HLSJS_TEST_BASE ? 'http://localhost:8000' : HLSJS_TEST_BASE,
path: '/tests/functional/auto/',
file: `index${HlsjsLightBuild ? '-light' : ''}.html`,
};
@@ -567,7 +638,7 @@ describe(`Testing hls.js playback in ${browserConfig.name} ${browserConfig.versi
};
}
if (!useSauce) {
if (!useRemote) {
// Configure webdriver for local testing
if (browserConfig.name === 'safari') {
browser = new webdriver.Builder()
@@ -587,6 +658,48 @@ describe(`Testing hls.js playback in ${browserConfig.name} ${browserConfig.versi
.withCapabilities(capabilities)
.build();
}
} else if (useBrowserStack) {
const bsLocalIdentifier =
process.env.BROWSERSTACK_LOCAL_IDENTIFIER || `local-${Date.now()}`;
const deviceName = process.env.DEVICE;
// BrowserStack does not use platformName; OS is set via bstack:options
delete capabilities.platformName;
const bstackOptions = {
buildName:
'HLSJS-' +
(process.env.BROWSERSTACK_BUILD_ID || `local-${Date.now()}`),
sessionName: capabilities.name,
local: 'true',
localIdentifier: bsLocalIdentifier,
userName: process.env.BROWSERSTACK_USERNAME,
accessKey: process.env.BROWSERSTACK_ACCESS_KEY,
};
if (deviceName) {
// Mobile device testing
const osVersion = browserConfig.platform.replace(
/^(Android|iOS)\s*/,
''
);
bstackOptions.deviceName = deviceName;
bstackOptions.os = browserConfig.platform.startsWith('iOS')
? 'ios'
: 'android';
bstackOptions.osVersion = osVersion;
delete capabilities.browserVersion;
} else {
// Desktop testing
const osMapping = mapOSToBrowserStack(browserConfig.platform);
bstackOptions.os = osMapping.os;
bstackOptions.osVersion = osMapping.osVersion;
}
capabilities['bstack:options'] = bstackOptions;
if (!process.env.BROWSERSTACK_LOCAL_IDENTIFIER) {
await startBrowserStackLocal(bsLocalIdentifier);
}
browser = new webdriver.Builder()
.usingServer('https://hub.browserstack.com/wd/hub')
.withCapabilities(capabilities)
.build();
} else {
if (process.env.SAUCE_TUNNEL_ID) {
capabilities['sauce:options'] = {
@@ -624,13 +737,20 @@ describe(`Testing hls.js playback in ${browserConfig.name} ${browserConfig.versi
browser.getSession(),
]);
console.log(`Retrieved session in ${Date.now() - start}ms`);
if (useSauce) {
if (useBrowserStack) {
console.log(
`BrowserStack session: https://automate.browserstack.com/sessions/${session.getId()}`
);
} else if (useSauce) {
console.log(`Job URL: https://saucelabs.com/jobs/${session.getId()}`);
} else {
console.log(`WebDriver SessionID: ${session.getId()}`);
}
});
} catch (err) {
if (useBrowserStack) {
await stopBrowserStackLocal();
}
await sauceDisconnect();
throw new Error(`failed setting up session: ${err}`);
}
@@ -646,7 +766,9 @@ describe(`Testing hls.js playback in ${browserConfig.name} ${browserConfig.versi
}
try {
await browser.get(testPageUrl);
await browser.manage().window().setRect(0, 0, 1200, 850);
if (!process.env.DEVICE) {
await browser.manage().window().setRect(0, 0, 1200, 850);
}
} catch (e) {
throw new Error('failed to open test page');
}
@@ -694,17 +816,27 @@ ${data.logs}
`);
failedUrls[url] = url in failedUrls ? failedUrls[url] + 1 : 1;
if (failed && useSauce) {
if (failed && useBrowserStack) {
browser.executeScript(
'browserstack_executor: {"action": "setSessionStatus", "arguments": {"status": "failed", "reason": "Test failed"}}'
);
} else if (failed && useSauce) {
browser.executeScript('sauce:job-result=failed');
}
}
});
after(async function () {
if (useSauce && this.currentTest && this.currentTest.parent) {
if (this.currentTest && this.currentTest.parent) {
const tests = this.currentTest.parent.tests;
if (tests && tests.length && tests.every((test) => test.isPassed())) {
browser.executeScript('sauce:job-result=passed');
if (useBrowserStack) {
browser.executeScript(
'browserstack_executor: {"action": "setSessionStatus", "arguments": {"status": "passed"}}'
);
} else if (useSauce) {
browser.executeScript('sauce:job-result=passed');
}
}
}
console.log('Quitting browser...');
@@ -713,6 +845,9 @@ ${data.logs}
if (Object.keys(failedUrls).length > 0) {
console.log(JSON.stringify(failedUrls, null, 2));
}
if (useBrowserStack) {
await stopBrowserStackLocal();
}
if (useSauce) {
await sauceDisconnect();
}
+2
View File
@@ -118,6 +118,7 @@ module.exports = {
url: 'https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s-fmp4/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8',
description: 'HLS fMP4 by Bitmovin',
abr: true,
skip_ua: ['safari'],
},
fmp4BitmovinHevc: {
url: 'https://bitmovin-a.akamaihd.net/content/dataset/multi-codec/hevc/stream_fmp4.m3u8',
@@ -277,6 +278,7 @@ module.exports = {
url: 'https://jvaryhlstests.blob.core.windows.net/hlstestdata/playlist_encrypted.m3u8',
description: 'aes-256 and aes-256-ctr full segment encryption',
abr: false,
skip_ua: ['safari'],
},
mpegTsHevcHls: {
url: 'https://devoldemar.github.io/streams/hls/bipbop/hevc.m3u8',