diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a829dff700..ac1fbc0557 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -177,7 +177,7 @@ jobs: # Enable core dumps in the subshell. - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run ctest" env: - PWMODE: "${{ matrix.mode }}" + PWTEST_MODE: "${{ matrix.mode }}" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -209,7 +209,7 @@ jobs: # Enable core dumps in the subshell. - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run test -- --tag=${{ matrix.browser }}" env: - PWVIDEO: 1 + PWTEST_VIDEO: 1 - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -272,7 +272,7 @@ jobs: # Enable core dumps in the subshell. - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run ctest" env: - PW_CHROMIUM_CHANNEL: "chrome" + PWTEST_CHANNEL: "chrome" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -301,7 +301,7 @@ jobs: - run: npm run ctest shell: bash env: - PW_CHROMIUM_CHANNEL: "chrome" + PWTEST_CHANNEL: "chrome" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) shell: bash @@ -327,7 +327,7 @@ jobs: - run: node lib/cli/cli install ffmpeg - run: npm run ctest env: - PW_CHROMIUM_CHANNEL: "chrome" + PWTEST_CHANNEL: "chrome" - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) - uses: actions/upload-artifact@v1 @@ -356,7 +356,7 @@ jobs: - run: npm run ctest shell: bash env: - PW_CHROMIUM_CHANNEL: "msedge" + PWTEST_CHANNEL: "msedge" - uses: actions/upload-artifact@v1 if: ${{ always() }} with: diff --git a/package.json b/package.json index 48fd01902a..2a68f6c896 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,8 @@ "ctest": "folio --config=tests/config/default.config.ts --tag=chromium", "ftest": "folio --config=tests/config/default.config.ts --tag=firefox", "wtest": "folio --config=tests/config/default.config.ts --tag=webkit", - "atest": "cross-env PW_ANDROID_TESTS=1 folio --config=tests/config/android.config.ts", - "etest": "cross-env PW_ELECTRON_TESTS=1 folio --config=tests/config/electron.config.ts", + "atest": "folio --config=tests/config/android.config.ts", + "etest": "folio --config=tests/config/electron.config.ts", "test": "folio --config=tests/config/default.config.ts", "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .", "tsc": "tsc -p .", diff --git a/packages/installation-tests/installation-tests.sh b/packages/installation-tests/installation-tests.sh index 485a2a9ba4..cf7c879688 100755 --- a/packages/installation-tests/installation-tests.sh +++ b/packages/installation-tests/installation-tests.sh @@ -453,7 +453,7 @@ function test_playwright_cli_codegen_should_work { npm install ${PLAYWRIGHT_TGZ} echo "Running playwright codegen" - OUTPUT=$(PWCLI_EXIT_FOR_TEST=1 xvfb-run --auto-servernum -- bash -c "npx playwright codegen") + OUTPUT=$(PWTEST_CLI_EXIT=1 xvfb-run --auto-servernum -- bash -c "npx playwright codegen") if [[ "${OUTPUT}" != *"chromium.launch"* ]]; then echo "ERROR: missing chromium.launch in the output" exit 1 @@ -464,7 +464,7 @@ function test_playwright_cli_codegen_should_work { fi echo "Running playwright codegen --target=python" - OUTPUT=$(PWCLI_EXIT_FOR_TEST=1 xvfb-run --auto-servernum -- bash -c "npx playwright codegen --target=python") + OUTPUT=$(PWTEST_CLI_EXIT=1 xvfb-run --auto-servernum -- bash -c "npx playwright codegen --target=python") if [[ "${OUTPUT}" != *"chromium.launch"* ]]; then echo "ERROR: missing chromium.launch in the output" exit 1 diff --git a/src/cli/cli.ts b/src/cli/cli.ts index c03ba11d78..217567c080 100755 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -348,7 +348,7 @@ async function openPage(context: BrowserContext, url: string | undefined): Promi } async function open(options: Options, url: string | undefined, language: string) { - const { context, launchOptions, contextOptions } = await launchContext(options, !!process.env.PWCLI_HEADLESS_FOR_TEST); + const { context, launchOptions, contextOptions } = await launchContext(options, !!process.env.PWTEST_CLI_HEADLESS); await context._enableRecorder({ language, launchOptions, @@ -357,12 +357,12 @@ async function open(options: Options, url: string | undefined, language: string) saveStorage: options.saveStorage, }); await openPage(context, url); - if (process.env.PWCLI_EXIT_FOR_TEST) + if (process.env.PWTEST_CLI_EXIT) await Promise.all(context.pages().map(p => p.close())); } async function codegen(options: Options, url: string | undefined, language: string, outputFile?: string) { - const { context, launchOptions, contextOptions } = await launchContext(options, !!process.env.PWCLI_HEADLESS_FOR_TEST); + const { context, launchOptions, contextOptions } = await launchContext(options, !!process.env.PWTEST_CLI_HEADLESS); if (process.env.PWTRACE) contextOptions._traceDir = path.join(process.cwd(), '.trace'); await context._enableRecorder({ @@ -375,7 +375,7 @@ async function codegen(options: Options, url: string | undefined, language: stri outputFile: outputFile ? path.resolve(outputFile) : undefined }); await openPage(context, url); - if (process.env.PWCLI_EXIT_FOR_TEST) + if (process.env.PWTEST_CLI_EXIT) await Promise.all(context.pages().map(p => p.close())); } diff --git a/src/server/dom.ts b/src/server/dom.ts index 7107972dde..2c58b358f2 100644 --- a/src/server/dom.ts +++ b/src/server/dom.ts @@ -93,7 +93,7 @@ export class FrameExecutionContext extends js.ExecutionContext { ${injectedScriptSource.source} return new pwExport( ${this.frame._page._delegate.rafCountForStablePosition()}, - ${!!process.env.PW_USE_TIMEOUT_FOR_RAF}, + ${!!process.env.PWTEST_USE_TIMEOUT_FOR_RAF}, [${custom.join(',\n')}] ); })(); diff --git a/src/server/supplements/recorder/recorderApp.ts b/src/server/supplements/recorder/recorderApp.ts index 021ce366cf..7688622b6f 100644 --- a/src/server/supplements/recorder/recorderApp.ts +++ b/src/server/supplements/recorder/recorderApp.ts @@ -99,15 +99,15 @@ export class RecorderApp extends EventEmitter { '--window-size=600,600', '--window-position=1280,10', ]; - if (process.env.PW_RECORDER_PORT) - args.push(`--remote-debugging-port=${process.env.PW_RECORDER_PORT}`); + if (process.env.PWTEST_RECORDER_PORT) + args.push(`--remote-debugging-port=${process.env.PWTEST_RECORDER_PORT}`); const context = await recorderPlaywright.chromium.launchPersistentContext(internalCallMetadata(), '', { channel: inspectedContext._browser.options.channel, sdkLanguage: inspectedContext._options.sdkLanguage, args, noDefaultViewport: true, - headless: !!process.env.PWCLI_HEADLESS_FOR_TEST || (isUnderTest() && !inspectedContext._browser.options.headful), - useWebSocket: !!process.env.PW_RECORDER_PORT + headless: !!process.env.PWTEST_CLI_HEADLESS || (isUnderTest() && !inspectedContext._browser.options.headful), + useWebSocket: !!process.env.PWTEST_RECORDER_PORT }); const controller = new ProgressController(internalCallMetadata(), context._browser); await controller.run(async progress => { @@ -145,7 +145,7 @@ export class RecorderApp extends EventEmitter { // Testing harness for runCLI mode. { - if (process.env.PWCLI_EXIT_FOR_TEST && sources.length) { + if (process.env.PWTEST_CLI_EXIT && sources.length) { process.stdout.write('\n-------------8<-------------\n'); process.stdout.write(sources[0].text); process.stdout.write('\n-------------8<-------------\n'); diff --git a/src/server/trace/viewer/traceViewer.ts b/src/server/trace/viewer/traceViewer.ts index da448bf00d..ce208d8ebb 100644 --- a/src/server/trace/viewer/traceViewer.ts +++ b/src/server/trace/viewer/traceViewer.ts @@ -131,7 +131,7 @@ class TraceViewer { sdkLanguage: 'javascript', args, noDefaultViewport: true, - headless: !!process.env.PWCLI_HEADLESS_FOR_TEST, + headless: !!process.env.PWTEST_CLI_HEADLESS, useWebSocket: isUnderTest() }); diff --git a/tests/browsercontext-viewport.spec.ts b/tests/browsercontext-viewport.spec.ts index de99df173b..f87d6fe70d 100644 --- a/tests/browsercontext-viewport.spec.ts +++ b/tests/browsercontext-viewport.spec.ts @@ -19,9 +19,9 @@ import { test as it, expect } from './config/pageTest'; import { test as browserTest } from './config/browserTest'; import { verifyViewport } from './config/utils'; -it.beforeEach(async () => { - it.skip(!!process.env.PW_ANDROID_TESTS, 'Default viewport is null'); - it.skip(!!process.env.PW_ELECTRON_TESTS, 'Default viewport is null'); +it.beforeEach(async ({ isElectron, isAndroid }) => { + it.skip(isAndroid, 'Default viewport is null'); + it.skip(isElectron, 'Default viewport is null'); }); it('should get the proper default viewport size', async ({page, server}) => { diff --git a/tests/chromium/chromium.spec.ts b/tests/chromium/chromium.spec.ts index 579238e647..f3f4bb44af 100644 --- a/tests/chromium/chromium.spec.ts +++ b/tests/chromium/chromium.spec.ts @@ -20,10 +20,10 @@ import { test as playwrightTest } from '../config/playwrightTest'; import http from 'http'; pageTest.describe('chromium', () => { - pageTest.beforeEach(async ({ browserName }) => { + pageTest.beforeEach(async ({ browserName, isElectron, isAndroid }) => { pageTest.skip(browserName !== 'chromium'); - pageTest.skip(!!process.env.PW_ANDROID_TESTS); - pageTest.skip(!!process.env.PW_ELECTRON_TESTS); + pageTest.skip(isAndroid); + pageTest.skip(isElectron); }); pageTest('should create a worker from a service worker', async ({page, server}) => { diff --git a/tests/chromium/js-coverage.spec.ts b/tests/chromium/js-coverage.spec.ts index 10d9355df6..720f2a6687 100644 --- a/tests/chromium/js-coverage.spec.ts +++ b/tests/chromium/js-coverage.spec.ts @@ -17,9 +17,9 @@ import { test as it, expect } from '../config/pageTest'; it.describe('JS Coverage', () => { - it.beforeEach(async ({ browserName }) => { + it.beforeEach(async ({ browserName, isElectron }) => { it.skip(browserName !== 'chromium'); - it.fixme(!!process.env.PW_ELECTRON_TESTS); + it.fixme(isElectron); }); it('should work', async function({page, server}) { diff --git a/tests/config/android.config.ts b/tests/config/android.config.ts index c618535376..24fb01c682 100644 --- a/tests/config/android.config.ts +++ b/tests/config/android.config.ts @@ -41,6 +41,6 @@ if (process.env.CI) { ]); } -const serverEnv = new ServerEnv(); +const serverEnv = new ServerEnv('10.0.2.2'); pageTest.runWith(folio.merge(serverEnv, new AndroidPageEnv()), { tag: 'android' }); androidTest.runWith(folio.merge(serverEnv, new AndroidEnv()), { tag: 'android' }); diff --git a/tests/config/androidEnv.ts b/tests/config/androidEnv.ts index 9d1f323eed..ed52642b81 100644 --- a/tests/config/androidEnv.ts +++ b/tests/config/androidEnv.ts @@ -54,6 +54,8 @@ export class AndroidEnv implements Env { isChromium: true, isFirefox: false, isWebKit: false, + isAndroid: true, + isElectron: false, browserName: 'chromium' as const, browserChannel: undefined, isWindows: os.platform() === 'win32', diff --git a/tests/config/browserEnv.ts b/tests/config/browserEnv.ts index 7c0640def3..d38c6e141c 100644 --- a/tests/config/browserEnv.ts +++ b/tests/config/browserEnv.ts @@ -16,7 +16,6 @@ import type { Env, WorkerInfo, TestInfo } from 'folio'; import type { Browser, BrowserContext, BrowserContextOptions, BrowserType, LaunchOptions } from '../../index'; -import { installCoverageHooks } from './coverage'; import { start } from '../../lib/outofprocess'; import { PlaywrightClient } from '../../lib/remote/playwrightClient'; import { removeFolders } from '../../lib/utils/utils'; @@ -37,7 +36,6 @@ type TestOptions = { mode: 'default' | 'driver' | 'service'; video?: boolean; traceDir?: string; - coverageBrowserName?: string; }; class DriverMode { @@ -105,7 +103,6 @@ export class PlaywrightEnv implements Env { protected _browserOptions: LaunchOptions; private _playwright: typeof import('../../index'); protected _browserType: BrowserType; - private _coverage: ReturnType | undefined; private _userDataDirs: string[] = []; private _persistentContext: BrowserContext | undefined; private _remoteServer: RemoteServer | undefined; @@ -121,7 +118,6 @@ export class PlaywrightEnv implements Env { } async beforeAll(workerInfo: WorkerInfo) { - this._coverage = installCoverageHooks(this._options.coverageBrowserName || this._browserName); require('../../lib/utils/utils').setUnderTest(); this._playwright = await this._mode.setup(workerInfo); this._browserType = this._playwright[this._browserName]; @@ -184,6 +180,8 @@ export class PlaywrightEnv implements Env { isChromium: this._browserName === 'chromium', isFirefox: this._browserName === 'firefox', isWebKit: this._browserName === 'webkit', + isAndroid: false, + isElectron: false, isWindows: os.platform() === 'win32', isMac: os.platform() === 'darwin', isLinux: os.platform() === 'linux', @@ -213,12 +211,6 @@ export class PlaywrightEnv implements Env { async afterAll(workerInfo: WorkerInfo) { await this._mode.teardown(); - const { coverage, uninstall } = this._coverage!; - uninstall(); - const coveragePath = path.join(__dirname, '..', 'coverage-report', workerInfo.workerIndex + '.json'); - const coverageJSON = Array.from(coverage.keys()).filter(key => coverage.get(key)); - await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true }); - await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8'); } } diff --git a/tests/config/cliEnv.ts b/tests/config/cliEnv.ts index 85ec2c25d4..db4c6def0c 100644 --- a/tests/config/cliEnv.ts +++ b/tests/config/cliEnv.ts @@ -31,7 +31,7 @@ export class CLIEnv extends PageEnv implements Env { this._port = 10907 + workerInfo.workerIndex * 2; this._server = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => this._handler(req, res)).listen(this._port); - process.env.PW_RECORDER_PORT = String(this._port + 1); + process.env.PWTEST_RECORDER_PORT = String(this._port + 1); } private _runCLI(args: string[]) { diff --git a/tests/config/cliTest.ts b/tests/config/cliTest.ts index 9ea2f01d9d..f74f428b0b 100644 --- a/tests/config/cliTest.ts +++ b/tests/config/cliTest.ts @@ -149,8 +149,8 @@ export class CLIMock { this.process = spawn('node', nodeArgs, { env: { ...process.env, - PWCLI_EXIT_FOR_TEST: '1', - PWCLI_HEADLESS_FOR_TEST: headless ? '1' : undefined, + PWTEST_CLI_EXIT: '1', + PWTEST_CLI_HEADLESS: headless ? '1' : undefined, }, stdio: 'pipe' }); diff --git a/tests/config/coverage.js b/tests/config/coverage.js index 2e80abcabb..9b6c3c7e5f 100644 --- a/tests/config/coverage.js +++ b/tests/config/coverage.js @@ -15,6 +15,9 @@ * limitations under the License. */ +const path = require('path'); +const fs = require('fs'); + /** * @param {Map} apiCoverage * @param {Object} events @@ -90,6 +93,35 @@ function installCoverageHooks(browserName) { return {coverage, uninstall}; } +class CoverageEnv { + /** + * @param {string} browserName + */ + constructor(browserName) { + this.browserName = browserName; + } + + /** + * @param {import('folio').WorkerInfo} workerInfo + */ + async beforeAll(workerInfo) { + this.coverage = installCoverageHooks(this.browserName); + } + + /** + * @param {import('folio').WorkerInfo} workerInfo + */ + async afterAll(workerInfo) { + const { coverage, uninstall } = this.coverage; + uninstall(); + const coveragePath = path.join(__dirname, '..', 'coverage-report', workerInfo.workerIndex + '.json'); + const coverageJSON = Array.from(coverage.keys()).filter(key => coverage.get(key)); + await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true }); + await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8'); + } +} + module.exports = { - installCoverageHooks + installCoverageHooks, + CoverageEnv }; diff --git a/tests/config/default.config.ts b/tests/config/default.config.ts index 6f0840c355..3823b7ec50 100644 --- a/tests/config/default.config.ts +++ b/tests/config/default.config.ts @@ -24,11 +24,12 @@ import { test as cliTest } from './cliTest'; import { PlaywrightEnv, BrowserEnv, PageEnv, BrowserName } from './browserEnv'; import { ServerEnv } from './serverEnv'; import { CLIEnv } from './cliEnv'; +import { CoverageEnv } from './coverage'; const config: folio.Config = { testDir: path.join(__dirname, '..'), outputDir: path.join(__dirname, '..', '..', 'test-results'), - timeout: process.env.PWVIDEO || process.env.PWTRACE ? 60000 : 30000, + timeout: process.env.PWTEST_VIDEO || process.env.PWTRACE ? 60000 : 30000, globalTimeout: 5400000, }; if (process.env.CI) { @@ -54,28 +55,27 @@ const getExecutablePath = (browserName: BrowserName) => { return process.env.WKPATH; }; -const serverEnv = new ServerEnv(); - const browsers = ['chromium', 'webkit', 'firefox'] as BrowserName[]; for (const browserName of browsers) { const executablePath = getExecutablePath(browserName); if (executablePath && (process.env.FOLIO_WORKER_INDEX === undefined || process.env.FOLIO_WORKER_INDEX === '')) console.error(`Using executable at ${executablePath}`); - const mode = (process.env.PWMODE || 'default') as ('default' | 'driver' | 'service'); + const mode = (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service'); const options = { mode, executablePath, traceDir: process.env.PWTRACE ? path.join(config.outputDir, 'trace') : undefined, headless: !process.env.HEADFUL, - channel: process.env.PW_CHROMIUM_CHANNEL as any, - video: !!process.env.PWVIDEO, + channel: process.env.PWTEST_CHANNEL as any, + video: !!process.env.PWTEST_VIDEO, }; - playwrightTest.runWith(folio.merge(serverEnv, new PlaywrightEnv(browserName, options)), { tag: browserName }); - playwrightSlowTest.runWith(folio.merge(serverEnv, new PlaywrightEnv(browserName, options)), { timeout: config.timeout * 3, tag: browserName }); - browserTest.runWith(folio.merge(serverEnv, new BrowserEnv(browserName, options)), { tag: browserName }); - browserSlowTest.runWith(folio.merge(serverEnv, new BrowserEnv(browserName, options)), { timeout: config.timeout * 3, tag: browserName }); - pageTest.runWith(folio.merge(serverEnv, new PageEnv(browserName, options)), { tag: browserName }); - contextTest.runWith(folio.merge(serverEnv, new PageEnv(browserName, options)), { tag: browserName }); + const commonEnv = folio.merge(new CoverageEnv(browserName), new ServerEnv()); + playwrightTest.runWith(folio.merge(commonEnv, new PlaywrightEnv(browserName, options)), { tag: browserName }); + playwrightSlowTest.runWith(folio.merge(commonEnv, new PlaywrightEnv(browserName, options)), { timeout: config.timeout * 3, tag: browserName }); + browserTest.runWith(folio.merge(commonEnv, new BrowserEnv(browserName, options)), { tag: browserName }); + browserSlowTest.runWith(folio.merge(commonEnv, new BrowserEnv(browserName, options)), { timeout: config.timeout * 3, tag: browserName }); + pageTest.runWith(folio.merge(commonEnv, new PageEnv(browserName, options)), { tag: browserName }); + contextTest.runWith(folio.merge(commonEnv, new PageEnv(browserName, options)), { tag: browserName }); if (mode !== 'service') - cliTest.runWith(folio.merge(serverEnv, new CLIEnv(browserName, options)), { tag: browserName }); + cliTest.runWith(folio.merge(commonEnv, new CLIEnv(browserName, options)), { tag: browserName }); } diff --git a/tests/config/electron.config.ts b/tests/config/electron.config.ts index 21ac676639..40f1a32774 100644 --- a/tests/config/electron.config.ts +++ b/tests/config/electron.config.ts @@ -20,6 +20,7 @@ import { test as electronTest } from './electronTest'; import { test as pageTest } from './pageTest'; import { ServerEnv } from './serverEnv'; import { ElectronEnv, ElectronPageEnv } from './electronEnv'; +import { CoverageEnv } from './coverage'; const config: folio.Config = { testDir: path.join(__dirname, '..'), @@ -42,5 +43,6 @@ if (process.env.CI) { } const serverEnv = new ServerEnv(); -electronTest.runWith(folio.merge(serverEnv, new ElectronEnv()), { tag: 'electron' }); -pageTest.runWith(folio.merge(serverEnv, new ElectronPageEnv()), { tag: 'electron' }); +const coverageEnv = new CoverageEnv('electron'); +electronTest.runWith(folio.merge(coverageEnv, serverEnv, new ElectronEnv()), { tag: 'electron' }); +pageTest.runWith(folio.merge(coverageEnv, serverEnv, new ElectronPageEnv()), { tag: 'electron' }); diff --git a/tests/config/electronEnv.ts b/tests/config/electronEnv.ts index 62be7ed37c..43ca57dd5a 100644 --- a/tests/config/electronEnv.ts +++ b/tests/config/electronEnv.ts @@ -27,7 +27,7 @@ export class ElectronEnv extends PlaywrightEnv implements Env protected _browserVersion: string; constructor() { - super('chromium', { mode: 'default', coverageBrowserName: 'electron' }); + super('chromium', { mode: 'default' }); // This env prevents 'Electron Security Policy' console message. process.env['ELECTRON_DISABLE_SECURITY_WARNINGS'] = 'true'; this._browserVersion = require('electron/package.json').version; @@ -62,6 +62,7 @@ export class ElectronEnv extends PlaywrightEnv implements Env }; return { ...result, + isElectron: true, electronApp: this._electronApp, newWindow: this._newWindow.bind(this), }; diff --git a/tests/config/pageTest.ts b/tests/config/pageTest.ts index d4b1a610b4..ee9c1c1041 100644 --- a/tests/config/pageTest.ts +++ b/tests/config/pageTest.ts @@ -33,6 +33,8 @@ export type CommonTestArgs = { isChromium: boolean; isFirefox: boolean; isWebKit: boolean; + isAndroid: boolean; + isElectron: boolean; isWindows: boolean; isMac: boolean; isLinux: boolean; diff --git a/tests/config/serverEnv.ts b/tests/config/serverEnv.ts index cdda097d15..9a9241ab98 100644 --- a/tests/config/serverEnv.ts +++ b/tests/config/serverEnv.ts @@ -25,17 +25,22 @@ export class ServerEnv implements Env { private _httpsServer: TestServer; private _socksServer: any; private _socksPort: number; + private _loopback: string | undefined; + + constructor(loopback?: string) { + this._loopback = loopback; + } async beforeAll(workerInfo: WorkerInfo) { const assetsPath = path.join(__dirname, '..', 'assets'); const cachedPath = path.join(__dirname, '..', 'assets', 'cached'); const port = 8907 + workerInfo.workerIndex * 3; - this._server = await TestServer.create(assetsPath, port); + this._server = await TestServer.create(assetsPath, port, this._loopback); this._server.enableHTTPCache(cachedPath); const httpsPort = port + 1; - this._httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort); + this._httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort, this._loopback); this._httpsServer.enableHTTPCache(cachedPath); this._socksServer = socks.createServer((info, accept, deny) => { diff --git a/tests/elementhandle-bounding-box.spec.ts b/tests/elementhandle-bounding-box.spec.ts index bb64c406c5..4fbbe4a51e 100644 --- a/tests/elementhandle-bounding-box.spec.ts +++ b/tests/elementhandle-bounding-box.spec.ts @@ -18,8 +18,8 @@ import { test as it, expect } from './config/pageTest'; import { test as browserTest } from './config/browserTest'; -it.beforeEach(async () => { - it.skip(!!process.env.PW_ANDROID_TESTS); +it.beforeEach(async ({ isAndroid }) => { + it.skip(isAndroid); }); it('should work', async ({ page, server, browserName, headful }) => { diff --git a/tests/elementhandle-owner-frame.spec.ts b/tests/elementhandle-owner-frame.spec.ts index c491f0763b..854ca736d9 100644 --- a/tests/elementhandle-owner-frame.spec.ts +++ b/tests/elementhandle-owner-frame.spec.ts @@ -73,8 +73,8 @@ it('should work for detached elements', async ({ page, server }) => { expect(await divHandle.ownerFrame()).toBe(page.mainFrame()); }); -it('should work for adopted elements', async ({ page, server }) => { - it.fixme(!!process.env.PW_ELECTRON_TESTS); +it('should work for adopted elements', async ({ page, server, isElectron }) => { + it.fixme(isElectron); await page.goto(server.EMPTY_PAGE); const [popup] = await Promise.all([ diff --git a/tests/elementhandle-query-selector.spec.ts b/tests/elementhandle-query-selector.spec.ts index 1e9e7e627c..bdbe3b3da1 100644 --- a/tests/elementhandle-query-selector.spec.ts +++ b/tests/elementhandle-query-selector.spec.ts @@ -34,8 +34,8 @@ it('should return null for non-existing element', async ({page, server}) => { expect(second).toBe(null); }); -it('should work for adopted elements', async ({page,server}) => { - it.fixme(!!process.env.PW_ELECTRON_TESTS); +it('should work for adopted elements', async ({page, server, isElectron}) => { + it.fixme(isElectron); await page.goto(server.EMPTY_PAGE); const [popup] = await Promise.all([ diff --git a/tests/elementhandle-screenshot.spec.ts b/tests/elementhandle-screenshot.spec.ts index 4396607196..53d751cb11 100644 --- a/tests/elementhandle-screenshot.spec.ts +++ b/tests/elementhandle-screenshot.spec.ts @@ -23,9 +23,9 @@ import path from 'path'; import fs from 'fs'; it.describe('element screenshot', () => { - it.beforeEach(async ({ browserName, headful }) => { + it.beforeEach(async ({ browserName, headful, isAndroid }) => { it.skip(browserName === 'firefox' && headful); - it.skip(!!process.env.PW_ANDROID_TESTS, 'Different dpr. Remove after using 1x scale for screenshots.'); + it.skip(isAndroid, 'Different dpr. Remove after using 1x scale for screenshots.'); }); it('should work', async ({page, server}) => { @@ -277,9 +277,9 @@ it.describe('element screenshot', () => { }); browserTest.describe('element sceenshot', () => { - browserTest.beforeEach(async ({ browserName, headful }) => { + browserTest.beforeEach(async ({ browserName, headful, isAndroid }) => { browserTest.skip(browserName === 'firefox' && headful); - browserTest.skip(!!process.env.PW_ANDROID_TESTS, 'Different dpr. Remove after using 1x scale for screenshots.'); + browserTest.skip(isAndroid, 'Different dpr. Remove after using 1x scale for screenshots.'); }); browserTest('should work with a mobile viewport', async ({browser, server, browserName}) => { diff --git a/tests/elementhandle-scroll-into-view.spec.ts b/tests/elementhandle-scroll-into-view.spec.ts index 21d41e2aef..7544577b71 100644 --- a/tests/elementhandle-scroll-into-view.spec.ts +++ b/tests/elementhandle-scroll-into-view.spec.ts @@ -17,8 +17,8 @@ import { test as it, expect } from './config/pageTest'; -it('should work', async ({ page, server }) => { - it.fixme(!!process.env.PW_ANDROID_TESTS); +it('should work', async ({ page, server, isAndroid }) => { + it.fixme(isAndroid); await page.goto(server.PREFIX + '/offscreenbuttons.html'); for (let i = 0; i < 11; ++i) { diff --git a/tests/emulation-focus.spec.ts b/tests/emulation-focus.spec.ts index f96937c551..92c3b5b6a6 100644 --- a/tests/emulation-focus.spec.ts +++ b/tests/emulation-focus.spec.ts @@ -23,8 +23,8 @@ it('should think that it is focused by default', async ({page}) => { expect(await page.evaluate('document.hasFocus()')).toBe(true); }); -it('should think that all pages are focused', async ({page}) => { - it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron'); +it('should think that all pages are focused', async ({page, isElectron}) => { + it.fixme(isElectron, 'BrowserContext.newPage does not work in Electron'); const page2 = await page.context().newPage(); expect(await page.evaluate('document.hasFocus()')).toBe(true); @@ -42,8 +42,8 @@ it('should focus popups by default', async ({page, server}) => { expect(await page.evaluate('document.hasFocus()')).toBe(true); }); -it('should provide target for keyboard events', async ({page, server}) => { - it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron'); +it('should provide target for keyboard events', async ({page, server, isElectron}) => { + it.fixme(isElectron, 'BrowserContext.newPage does not work in Electron'); const page2 = await page.context().newPage(); await Promise.all([ @@ -67,8 +67,8 @@ it('should provide target for keyboard events', async ({page, server}) => { expect(results).toEqual([text, text2]); }); -it('should not affect mouse event target page', async ({page, server}) => { - it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron'); +it('should not affect mouse event target page', async ({page, server, isElectron}) => { + it.fixme(isElectron, 'BrowserContext.newPage does not work in Electron'); const page2 = await page.context().newPage(); function clickCounter() { @@ -91,8 +91,8 @@ it('should not affect mouse event target page', async ({page, server}) => { expect(counters).toEqual([1,1]); }); -it('should change document.activeElement', async ({page, server}) => { - it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron'); +it('should change document.activeElement', async ({page, server, isElectron}) => { + it.fixme(isElectron, 'BrowserContext.newPage does not work in Electron'); const page2 = await page.context().newPage(); await Promise.all([ @@ -110,10 +110,10 @@ it('should change document.activeElement', async ({page, server}) => { expect(active).toEqual(['INPUT', 'TEXTAREA']); }); -it('should not affect screenshots', async ({page, server, browserName, headful}) => { +it('should not affect screenshots', async ({page, server, browserName, headful, isElectron, isAndroid}) => { it.skip(browserName === 'firefox' && headful); - it.skip(!!process.env.PW_ANDROID_TESTS); - it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron'); + it.skip(isAndroid); + it.fixme(isElectron, 'BrowserContext.newPage does not work in Electron'); // Firefox headful produces a different image. const page2 = await page.context().newPage(); diff --git a/tests/frame-evaluate.spec.ts b/tests/frame-evaluate.spec.ts index ea707123cf..308104c347 100644 --- a/tests/frame-evaluate.spec.ts +++ b/tests/frame-evaluate.spec.ts @@ -96,9 +96,9 @@ it('should allow cross-frame element handles', async ({ page, server }) => { expect(result.trim()).toBe('
Hi, I\'m frame
'); }); -it('should not allow cross-frame element handles when frames do not script each other', async ({ page, server }) => { - it.skip(!!process.env.PW_ANDROID_TESTS); - it.fixme(!!process.env.PW_ELECTRON_TESTS); +it('should not allow cross-frame element handles when frames do not script each other', async ({ page, server, isElectron, isAndroid }) => { + it.skip(isAndroid, 'No cross-process on Android'); + it.fixme(isElectron); await page.goto(server.EMPTY_PAGE); const frame = await attachFrame(page, 'frame1', server.CROSS_PROCESS_PREFIX + '/empty.html'); diff --git a/tests/frame-goto.spec.ts b/tests/frame-goto.spec.ts index 6a71c9b9b2..8ce31437a7 100644 --- a/tests/frame-goto.spec.ts +++ b/tests/frame-goto.spec.ts @@ -40,8 +40,8 @@ it('should reject when frame detaches', async ({page, server}) => { expect(error.message).toContain('frame was detached'); }); -it('should continue after client redirect', async ({page, server}) => { - it.fixme(!!process.env.PW_ANDROID_TESTS); +it('should continue after client redirect', async ({page, server, isAndroid}) => { + it.fixme(isAndroid); server.setRoute('/frames/script.js', () => {}); const url = server.PREFIX + '/frames/child-redirect.html'; diff --git a/tests/frame-hierarchy.spec.ts b/tests/frame-hierarchy.spec.ts index c4cc92f11b..7752a08044 100644 --- a/tests/frame-hierarchy.spec.ts +++ b/tests/frame-hierarchy.spec.ts @@ -35,8 +35,8 @@ function dumpFrames(frame: Frame, indentation: string = ''): string[] { return result; } -it('should handle nested frames', async ({page, server}) => { - it.skip(!!process.env.PW_ANDROID_TESTS); +it('should handle nested frames', async ({page, server, isAndroid}) => { + it.skip(isAndroid, 'No cross-process on Android'); await page.goto(server.PREFIX + '/frames/nested-frames.html'); expect(dumpFrames(page.mainFrame())).toEqual([ @@ -154,8 +154,8 @@ it('should report frame from-inside shadow DOM', async ({page, server}) => { expect(page.frames()[1].url()).toBe(server.EMPTY_PAGE); }); -it('should report frame.name()', async ({page, server}) => { - it.fixme(!!process.env.PW_ELECTRON_TESTS); +it('should report frame.name()', async ({page, server, isElectron}) => { + it.fixme(isElectron); await attachFrame(page, 'theFrameId', server.EMPTY_PAGE); await page.evaluate(url => { @@ -178,8 +178,8 @@ it('should report frame.parent()', async ({page, server}) => { expect(page.frames()[2].parentFrame()).toBe(page.mainFrame()); }); -it('should report different frame instance when frame re-attaches', async ({page, server}) => { - it.fixme(!!process.env.PW_ELECTRON_TESTS); +it('should report different frame instance when frame re-attaches', async ({page, server, isElectron}) => { + it.fixme(isElectron); const frame1 = await attachFrame(page, 'frame1', server.EMPTY_PAGE); await page.evaluate(() => { diff --git a/tests/interception.spec.ts b/tests/interception.spec.ts index 389dac0007..2b61d64c0e 100644 --- a/tests/interception.spec.ts +++ b/tests/interception.spec.ts @@ -44,8 +44,8 @@ browserTest('should work with ignoreHTTPSErrors', async ({browser, httpsServer}) await context.close(); }); -it('should intercept after a service worker', async ({page, server}) => { - it.skip(!!process.env.PW_ANDROID_TESTS); +it('should intercept after a service worker', async ({page, server, isAndroid}) => { + it.skip(isAndroid); await page.goto(server.PREFIX + '/serviceworkers/fetchdummy/sw.html'); await page.evaluate(() => window['activationPromise']); @@ -90,8 +90,8 @@ it('should work with glob', async () => { expect(globToRegex('**/*.{png,jpg,jpeg}').test('https://localhost:8080/c.css')).toBeFalsy(); }); -it('should intercept network activity from worker', async function({page, server}) { - it.skip(!!process.env.PW_ANDROID_TESTS); +it('should intercept network activity from worker', async function({page, server, isAndroid}) { + it.skip(isAndroid); await page.goto(server.EMPTY_PAGE); server.setRoute('/data_for_worker', (req, res) => res.end('failed to intercept')); @@ -111,9 +111,9 @@ it('should intercept network activity from worker', async function({page, server expect(msg.text()).toBe('intercepted'); }); -it('should intercept network activity from worker 2', async function({page, server}) { - it.skip(!!process.env.PW_ANDROID_TESTS); - it.fixme(!!process.env.PW_ELECTRON_TESTS); +it('should intercept network activity from worker 2', async function({page, server, isElectron, isAndroid}) { + it.skip(isAndroid); + it.fixme(isElectron); const url = server.PREFIX + '/worker/worker.js'; await page.route(url, route => { @@ -130,8 +130,8 @@ it('should intercept network activity from worker 2', async function({page, serv expect(msg.text()).toBe('intercepted'); }); -it('should work with regular expression passed from a different context', async ({page, server}) => { - it.skip(!!process.env.PW_ELECTRON_TESTS); +it('should work with regular expression passed from a different context', async ({page, server, isElectron}) => { + it.skip(isElectron); const ctx = vm.createContext(); const regexp = vm.runInContext('new RegExp("empty\\.html")', ctx); diff --git a/tests/network-post-data.spec.ts b/tests/network-post-data.spec.ts index 169bb52152..bca2fab892 100644 --- a/tests/network-post-data.spec.ts +++ b/tests/network-post-data.spec.ts @@ -16,8 +16,8 @@ import { test as it, expect } from './config/pageTest'; -it.beforeEach(async () => { - it.skip(!!process.env.PW_ANDROID_TESTS); +it.beforeEach(async ({ isAndroid }) => { + it.fixme(isAndroid, 'Post data does not work'); }); it('should return correct postData buffer for utf-8 body', async ({page, server}) => { diff --git a/tests/page-add-script-tag.spec.ts b/tests/page-add-script-tag.spec.ts index 80a571ee93..8e5a34b78d 100644 --- a/tests/page-add-script-tag.spec.ts +++ b/tests/page-add-script-tag.spec.ts @@ -98,8 +98,8 @@ it('should throw when added with content to the CSP page', async ({page, server} expect(error).toBeTruthy(); }); -it('should throw when added with URL to the CSP page', async ({page, server}) => { - it.skip(!!process.env.PW_ANDROID_TESTS); +it('should throw when added with URL to the CSP page', async ({page, server, isAndroid}) => { + it.skip(isAndroid, 'No cross-process on Android'); await page.goto(server.PREFIX + '/csp.html'); let error = null; diff --git a/tests/page-add-style-tag.spec.ts b/tests/page-add-style-tag.spec.ts index 6b6bdd939a..73a51959b0 100644 --- a/tests/page-add-style-tag.spec.ts +++ b/tests/page-add-style-tag.spec.ts @@ -76,8 +76,8 @@ it('should throw when added with content to the CSP page', async ({page, server} expect(error).toBeTruthy(); }); -it('should throw when added with URL to the CSP page', async ({page, server}) => { - it.skip(!!process.env.PW_ANDROID_TESTS); +it('should throw when added with URL to the CSP page', async ({page, server, isAndroid}) => { + it.skip(isAndroid, 'No cross-process on Android'); await page.goto(server.PREFIX + '/csp.html'); let error = null; diff --git a/tests/page-autowaiting-basic.spec.ts b/tests/page-autowaiting-basic.spec.ts index 01306c16a2..56f7cbf34f 100644 --- a/tests/page-autowaiting-basic.spec.ts +++ b/tests/page-autowaiting-basic.spec.ts @@ -28,8 +28,8 @@ function initServer(server: TestServer): string[] { return messages; } -it.beforeEach(async () => { - it.skip(!!process.env.PW_ANDROID_TESTS, 'Too flaky on Android'); +it.beforeEach(async ({ isAndroid }) => { + it.skip(isAndroid, 'Too flaky on Android'); }); it('should await navigation when clicking anchor', async ({page, server}) => { diff --git a/tests/page-basic.spec.ts b/tests/page-basic.spec.ts index 5704593b4d..5c839fa637 100644 --- a/tests/page-basic.spec.ts +++ b/tests/page-basic.spec.ts @@ -32,8 +32,8 @@ it('should set the page close state', async ({page}) => { expect(page.isClosed()).toBe(true); }); -it('should pass page to close event', async ({page}) => { - it.fixme(!!process.env.PW_ANDROID_TESTS); +it('should pass page to close event', async ({page, isAndroid}) => { + it.fixme(isAndroid); const [closedPage] = await Promise.all([ page.waitForEvent('close'), @@ -121,8 +121,8 @@ it('should pass self as argument to load event', async ({page}) => { expect(eventArg).toBe(page); }); -it('should fail with error upon disconnect', async ({page}) => { - it.fixme(!!process.env.PW_ANDROID_TESTS); +it('should fail with error upon disconnect', async ({page, isAndroid}) => { + it.fixme(isAndroid); let error; const waitForPromise = page.waitForEvent('download').catch(e => error = e); @@ -131,8 +131,8 @@ it('should fail with error upon disconnect', async ({page}) => { expect(error.message).toContain('Page closed'); }); -it('page.url should work', async ({page, server}) => { - it.fixme(!!process.env.PW_ELECTRON_TESTS); +it('page.url should work', async ({page, server, isElectron}) => { + it.fixme(isElectron); expect(page.url()).toBe('about:blank'); await page.goto(server.EMPTY_PAGE); @@ -182,9 +182,9 @@ it('page.frame should respect url', async function({page, server}) { expect(page.frame({ url: /empty/ }).url()).toBe(server.EMPTY_PAGE); }); -it('should have sane user agent', async ({page, isChromium, isFirefox}) => { - it.skip(!!process.env.PW_ANDROID_TESTS); - it.skip(!!process.env.PW_ELECTRON_TESTS); +it('should have sane user agent', async ({page, isChromium, isFirefox, isElectron, isAndroid}) => { + it.skip(isAndroid); + it.skip(isElectron); const userAgent = await page.evaluate(() => navigator.userAgent); const [ diff --git a/tests/page-click-scroll.spec.ts b/tests/page-click-scroll.spec.ts index 6972987086..a0a27b1144 100644 --- a/tests/page-click-scroll.spec.ts +++ b/tests/page-click-scroll.spec.ts @@ -16,9 +16,9 @@ import { test as it } from './config/pageTest'; -it('should not hit scroll bar', async ({page, server, isWebKit, platform}) => { +it('should not hit scroll bar', async ({page, isAndroid, isWebKit, platform}) => { it.fixme(isWebKit && platform === 'darwin'); - it.skip(!!process.env.PW_ANDROID_TESTS); + it.skip(isAndroid); await page.setContent(`