From 6ffdd4dfa1b8daf7bc43abd910b738ffee3ace44 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 28 Aug 2020 00:32:00 -0700 Subject: [PATCH] feat(testrunner): allow unexpected passes (#3665) --- .github/workflows/tests.yml | 9 --- test-runner/src/builtin.fixtures.ts | 1 + test-runner/src/reporters/base.ts | 63 +++++++++-------- test-runner/src/reporters/dot.ts | 4 +- test-runner/src/reporters/list.ts | 14 ++-- test-runner/src/reporters/pytest.ts | 13 ++-- test-runner/src/runner.ts | 18 +++-- test-runner/src/spec.ts | 10 ++- test-runner/src/test.ts | 29 +++++--- test-runner/src/testRunner.ts | 21 +++--- test-runner/src/worker.ts | 4 +- test-runner/test/assets/expected-failure.js | 24 +++++++ test-runner/test/assets/nested-skip.js | 22 ++++++ test-runner/test/assets/one-timeout.js | 20 ++++++ test-runner/test/assets/unexpected-pass.js | 20 ++++++ test-runner/test/exit-code.spec.ts | 64 +++++++++++++++++- .../firefox/screenshot-webgl.png | Bin 0 -> 4016 bytes .../webkit/screenshot-webgl.png | Bin 0 -> 3489 bytes test/browsercontext-credentials.spec.ts | 2 +- test/browsercontext-locale.spec.ts | 2 +- test/browsercontext-page-event.spec.ts | 4 +- test/browsercontext-timezone-id.spec.ts | 2 +- test/browsertype-connect.spec.ts | 2 +- test/browsertype-launch.spec.ts | 2 +- test/capabilities.spec.ts | 2 +- test/chromium/oopif.spec.ts | 2 +- test/click-react.spec.ts | 6 +- test/click.spec.ts | 2 +- test/coverage.js | 2 +- test/defaultbrowsercontext-2.spec.ts | 4 +- test/dialog.spec.ts | 2 +- test/download.spec.ts | 4 +- test/elementhandle-owner-frame.spec.ts | 3 +- test/elementhandle-screenshot.spec.ts | 2 +- ...ementhandle-wait-for-element-state.spec.ts | 1 + test/frame-evaluate.spec.ts | 2 +- test/frame-hierarchy.spec.ts | 2 +- test/headful.spec.ts | 2 +- test/mouse.spec.ts | 2 +- test/page-add-script-tag.spec.ts | 2 +- test/page-emulate-media.spec.ts | 2 +- test/page-evaluate.spec.ts | 6 +- test/page-event-crash.spec.ts | 12 ++-- test/page-screenshot.spec.ts | 2 +- test/page-wait-for-navigation.spec.ts | 2 +- test/permissions.spec.ts | 2 +- test/screencast.spec.ts | 10 +-- 47 files changed, 299 insertions(+), 127 deletions(-) create mode 100644 test-runner/test/assets/expected-failure.js create mode 100644 test-runner/test/assets/nested-skip.js create mode 100644 test-runner/test/assets/one-timeout.js create mode 100644 test-runner/test/assets/unexpected-pass.js create mode 100644 test/__snapshots__/page-screenshot/firefox/screenshot-webgl.png create mode 100644 test/__snapshots__/page-screenshot/webkit/screenshot-webgl.png diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 572898bc6f..2f2ce2b8a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,8 +40,6 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json && npm run coverage" env: BROWSER: ${{ matrix.browser }} - DEBUG: "pw:*,-pw:wrapped*,-pw:test*" - DEBUG_FILE: "test-results/debug.log" PWRUNNER_JSON_REPORT: "test-results/report.json" - uses: actions/upload-artifact@v1 if: always() @@ -67,8 +65,6 @@ jobs: - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json env: BROWSER: ${{ matrix.browser }} - DEBUG: "pw:*,-pw:wrapped*,-pw:test*" - DEBUG_FILE: "test-results/debug.log" PWRUNNER_JSON_REPORT: "test-results/report.json" - uses: actions/upload-artifact@v1 if: ${{ always() }} @@ -98,8 +94,6 @@ jobs: shell: bash env: BROWSER: ${{ matrix.browser }} - DEBUG: "pw:*,-pw:wrapped*,-pw:test*" - DEBUG_FILE: "test-results/debug.log" PWRUNNER_JSON_REPORT: "test-results/report.json" - uses: actions/upload-artifact@v1 if: ${{ always() }} @@ -152,7 +146,6 @@ jobs: env: BROWSER: ${{ matrix.browser }} HEADLESS: "false" - DEBUG_FILE: "test-results/debug.log" PWRUNNER_JSON_REPORT: "test-results/report.json" - uses: actions/upload-artifact@v1 if: ${{ always() }} @@ -184,8 +177,6 @@ jobs: - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json" env: BROWSER: ${{ matrix.browser }} - DEBUG: "pw:*,-pw:wrapped*,-pw:test*" - DEBUG_FILE: "test-results/debug.log" PWWIRE: true PWRUNNER_JSON_REPORT: "test-results/report.json" - uses: actions/upload-artifact@v1 diff --git a/test-runner/src/builtin.fixtures.ts b/test-runner/src/builtin.fixtures.ts index 0e59bbcda1..83d3d1c856 100644 --- a/test-runner/src/builtin.fixtures.ts +++ b/test-runner/src/builtin.fixtures.ts @@ -32,6 +32,7 @@ declare global { type ItFunction = ((name: string, inner: (state: STATE) => Promise | void) => void) & { fail(condition: boolean): ItFunction; + fixme(condition: boolean): ItFunction; flaky(condition: boolean): ItFunction; skip(condition: boolean): ItFunction; slow(): ItFunction; diff --git a/test-runner/src/reporters/base.ts b/test-runner/src/reporters/base.ts index 01e7095d8d..eee022e09d 100644 --- a/test-runner/src/reporters/base.ts +++ b/test-runner/src/reporters/base.ts @@ -30,10 +30,9 @@ const stackUtils = new StackUtils(); export class BaseReporter implements Reporter { skipped: Test[] = []; - passed: Test[] = []; + asExpected: Test[] = []; + unexpected = new Set(); flaky: Test[] = []; - failed: Test[] = []; - timedOut: Test[] = []; duration = 0; startTime: number; config: RunnerConfig; @@ -66,28 +65,24 @@ export class BaseReporter implements Reporter { } onTestEnd(test: Test, result: TestResult) { - switch (result.status) { - case 'skipped': { - this.skipped.push(test); - return; - } - case 'passed': - if (test.results.length === 1) - this.passed.push(test); - else - this.flaky.push(test); - return; - case 'failed': - // Fall through. - case 'timedOut': { - if (test.results.length === this.config.retries + 1) { - if (result.status === 'timedOut') - this.timedOut.push(test); - else - this.failed.push(test); - } - return; + if (result.status === 'skipped') { + this.skipped.push(test); + return; + } + + if (result.status === result.expectedStatus) { + if (test.results.length === 1) { + // as expected from the first attempt + this.asExpected.push(test); + } else { + // as expected after unexpected -> flaky. + this.flaky.push(test); } + return; + } + if (result.status === 'passed' || result.status === 'timedOut' || test.results.length === this.config.retries + 1) { + // We made as many retries as we could, still failing. + this.unexpected.add(test); } } @@ -98,15 +93,16 @@ export class BaseReporter implements Reporter { epilogue() { console.log(''); - console.log(colors.green(` ${this.passed.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); + console.log(colors.green(` ${this.asExpected.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); if (this.skipped.length) console.log(colors.yellow(` ${this.skipped.length} skipped`)); - if (this.failed.length) { - console.log(colors.red(` ${this.failed.length} failed`)); + const filteredUnexpected = [...this.unexpected].filter(t => !t._hasResultWithStatus('timedOut')); + if (filteredUnexpected.length) { + console.log(colors.red(` ${filteredUnexpected.length} failed`)); console.log(''); - this._printFailures(this.failed); + this._printFailures(filteredUnexpected); } if (this.flaky.length) { @@ -115,11 +111,13 @@ export class BaseReporter implements Reporter { this._printFailures(this.flaky); } - if (this.timedOut.length) { - console.log(colors.red(` ${this.timedOut.length} timed out`)); + const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut')); + if (timedOut.length) { + console.log(colors.red(` ${timedOut.length} timed out`)); console.log(''); - this._printFailures(this.timedOut); + this._printFailures(timedOut); } + console.log(''); } private _printFailures(failures: Test[]) { @@ -131,7 +129,8 @@ export class BaseReporter implements Reporter { formatFailure(test: Test, index?: number): string { const tokens: string[] = []; const relativePath = path.relative(process.cwd(), test.file); - const header = ` ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}`; + const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : ''; + const header = ` ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}${passedUnexpectedlySuffix}`; tokens.push(colors.bold(colors.red(header))); for (const result of test.results) { if (result.status === 'passed') diff --git a/test-runner/src/reporters/dot.ts b/test-runner/src/reporters/dot.ts index dc98f00904..b065ec1e1e 100644 --- a/test-runner/src/reporters/dot.ts +++ b/test-runner/src/reporters/dot.ts @@ -23,8 +23,8 @@ class DotReporter extends BaseReporter { super.onTestEnd(test, result); switch (result.status) { case 'skipped': process.stdout.write(colors.yellow('∘')); break; - case 'passed': process.stdout.write(colors.green('·')); break; - case 'failed': process.stdout.write(colors.red(test.results.length > 1 ? '' + test.results.length : 'F')); break; + case 'passed': process.stdout.write(result.status === result.expectedStatus ? colors.green('·') : colors.red('P')); break; + case 'failed': process.stdout.write(result.status === result.expectedStatus ? colors.green('f') : colors.red('F')); break; case 'timedOut': process.stdout.write(colors.red('T')); break; } } diff --git a/test-runner/src/reporters/list.ts b/test-runner/src/reporters/list.ts index ab37cdc6d5..4d485cd184 100644 --- a/test-runner/src/reporters/list.ts +++ b/test-runner/src/reporters/list.ts @@ -35,12 +35,14 @@ class ListReporter extends BaseReporter { onTestEnd(test: Test, result: TestResult) { super.onTestEnd(test, result); let text = ''; - switch (result.status) { - case 'skipped': text = colors.green(' - ') + colors.cyan(test.fullTitle()); break; - case 'passed': text = '\u001b[2K\u001b[0G' + colors.green(' ✓ ') + colors.gray(test.fullTitle()); break; - case 'failed': - // fall through - case 'timedOut': text = '\u001b[2K\u001b[0G' + colors.red(` ${++this._failure}) ` + test.fullTitle()); break; + if (result.status === 'skipped') { + text = colors.green(' - ') + colors.cyan(test.fullTitle()); + } else { + const statusMark = result.status === 'passed' ? colors.green(' ✓ ') : colors.red(' x '); + if (result.status === result.expectedStatus) + text = '\u001b[2K\u001b[0G' + statusMark + colors.gray(test.fullTitle()); + else + text = '\u001b[2K\u001b[0G' + colors.red(` ${++this._failure}) ` + test.fullTitle()); } process.stdout.write(text + '\n'); } diff --git a/test-runner/src/reporters/pytest.ts b/test-runner/src/reporters/pytest.ts index 9e4debb979..6e967085ef 100644 --- a/test-runner/src/reporters/pytest.ts +++ b/test-runner/src/reporters/pytest.ts @@ -149,14 +149,15 @@ class PytestReporter extends BaseReporter { } const status = []; - if (this.passed.length) - status.push(colors.green(`${this.passed.length} passed`)); + if (this.asExpected.length) + status.push(colors.green(`${this.asExpected.length} as expected`)); if (this.skipped.length) status.push(colors.yellow(`${this.skipped.length} skipped`)); - if (this.failed.length) - status.push(colors.red(`${this.failed.length} failed`)); - if (this.timedOut.length) - status.push(colors.red(`${this.timedOut.length} timed out`)); + const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut')); + if (this.unexpected.size - timedOut.length) + status.push(colors.red(`${this.unexpected.size - timedOut.length} unexpected failures`)); + if (timedOut.length) + status.push(colors.red(`${timedOut.length} timed out`)); status.push(colors.dim(`(${milliseconds(Date.now() - this.startTime)})`)); for (let i = lines.length; i < this._visibleRows; ++i) diff --git a/test-runner/src/runner.ts b/test-runner/src/runner.ts index 65571b6edc..cdd2093af9 100644 --- a/test-runner/src/runner.ts +++ b/test-runner/src/runner.ts @@ -103,7 +103,11 @@ export class Runner { let doneCallback; const result = new Promise(f => doneCallback = f); worker.once('done', params => { - if (!params.failedTestId && !params.fatalError) { + // We won't file remaining if: + // - there are no remaining + // - we are here not because something failed + // - no unrecoverable worker error + if (!params.remaining.length && !params.failedTestId && !params.fatalError) { this._workerAvailable(worker); doneCallback(); return; @@ -112,8 +116,8 @@ export class Runner { // When worker encounters error, we will restart it. this._restartWorker(worker); - // In case of fatal error without test id, we are done with the entry. - if (params.fatalError && !params.failedTestId) { + // In case of fatal error, we are done with the entry. + if (params.fatalError) { // Report all the tests are failing with this error. for (const id of entry.ids) { const { test, result } = this._testById.get(id); @@ -127,13 +131,16 @@ export class Runner { } const remaining = params.remaining; - if (this._config.retries) { + + // Only retry expected failures, not passes and only if the test failed. + if (this._config.retries && params.failedTestId) { const pair = this._testById.get(params.failedTestId); - if (pair.test.results.length < this._config.retries + 1) { + if (pair.result.expectedStatus === 'passed' && pair.test.results.length < this._config.retries + 1) { pair.result = pair.test._appendResult(); remaining.unshift(pair.test._id); } } + if (remaining.length) this._queue.unshift({ ...entry, ids: remaining }); @@ -169,6 +176,7 @@ export class Runner { const { test } = this._testById.get(params.id); test._skipped = params.skipped; test._flaky = params.flaky; + test._expectedStatus = params.expectedStatus; this._reporter.onTestBegin(test); }); worker.on('testEnd', params => { diff --git a/test-runner/src/spec.ts b/test-runner/src/spec.ts index bcdbeecd11..f9a70a29ac 100644 --- a/test-runner/src/spec.ts +++ b/test-runner/src/spec.ts @@ -53,7 +53,7 @@ export function spec(suite: Suite, file: string, timeout: number): () => void { const suites = [suite]; suite.file = file; - const it = specBuilder(['skip', 'fail', 'slow', 'only', 'flaky'], (specs, title, fn) => { + const it = specBuilder(['skip', 'fixme', 'fail', 'slow', 'only', 'flaky'], (specs, title, fn) => { const suite = suites[0]; const test = new Test(title, fn); test.file = file; @@ -65,8 +65,10 @@ export function spec(suite: Suite, file: string, timeout: number): () => void { test.only = true; if (!only && specs.skip && specs.skip[0]) test._skipped = true; - if (!only && specs.fail && specs.fail[0]) + if (!only && specs.fixme && specs.fixme[0]) test._skipped = true; + if (specs.fail && specs.fail[0]) + test._expectedStatus = 'failed'; if (specs.flaky && specs.flaky[0]) test._flaky = true; suite._addTest(test); @@ -81,7 +83,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void { if (only) child.only = true; if (!only && specs.skip && specs.skip[0]) - child.skipped = true; + child._skipped = true; + if (!only && specs.fixme && specs.fixme[0]) + child._skipped = true; suites.unshift(child); fn(); suites.shift(); diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts index 70a36fa5ca..4a681dcef1 100644 --- a/test-runner/src/test.ts +++ b/test-runner/src/test.ts @@ -16,6 +16,8 @@ export type Configuration = { name: string, value: string }[]; +type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; + export class Test { suite: Suite; title: string; @@ -27,10 +29,13 @@ export class Test { results: TestResult[] = []; _id: string; + // Skipped & flaky are resolved based on options in worker only + // We will compute them there and send to the runner (front-end) _skipped = false; _flaky = false; _overriddenFn: Function; _startTime: number; + _expectedStatus: TestStatus = 'passed'; constructor(title: string, fn: Function) { this.title = title; @@ -48,7 +53,7 @@ export class Test { _appendResult(): TestResult { const result: TestResult = { duration: 0, - status: 'none', + expectedStatus: 'passed', stdout: [], stderr: [], data: {} @@ -58,17 +63,21 @@ export class Test { } _ok(): boolean { - if (this._skipped) + if (this._skipped || this.suite._isSkipped()) return true; - const hasFailedResults = !!this.results.find(r => r.status !== 'passed' && r.status !== 'skipped'); + const hasFailedResults = !!this.results.find(r => r.status !== r.expectedStatus); if (!hasFailedResults) return true; if (!this._flaky) return false; - const hasPassedResults = !!this.results.find(r => r.status === 'passed'); + const hasPassedResults = !!this.results.find(r => r.status === r.expectedStatus); return hasPassedResults; } + _hasResultWithStatus(status: TestStatus): boolean { + return !!this.results.find(r => r.status === status); + } + _clone(): Test { const test = new Test(this.title, this.fn); test.suite = this.suite; @@ -83,7 +92,8 @@ export class Test { export type TestResult = { duration: number; - status: 'none' | 'passed' | 'failed' | 'timedOut' | 'skipped'; + status?: TestStatus; + expectedStatus: TestStatus; error?: any; stdout: (string | Buffer)[]; stderr: (string | Buffer)[]; @@ -96,9 +106,12 @@ export class Suite { suites: Suite[] = []; tests: Test[] = []; only = false; - skipped = false; file: string; configuration: Configuration; + + // Skipped & flaky are resolved based on options in worker only + // We will compute them there and send to the runner (front-end) + _skipped = false; _configurationString: string; _hooks: { type: string, fn: Function } [] = []; @@ -124,7 +137,7 @@ export class Suite { } _isSkipped(): boolean { - return this.skipped || (this.parent && this.parent._isSkipped()); + return this._skipped || (this.parent && this.parent._isSkipped()); } _addTest(test: Test) { @@ -163,7 +176,7 @@ export class Suite { const suite = new Suite(this.title); suite.only = this.only; suite.file = this.file; - suite.skipped = this.skipped; + suite._skipped = this._skipped; return suite; } diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts index 2f61e1ff3a..d5d979190c 100644 --- a/test-runner/src/testRunner.ts +++ b/test-runner/src/testRunner.ts @@ -46,7 +46,6 @@ export class TestRunner extends EventEmitter { private _ids: Set; private _remaining: Set; private _trialRun: any; - private _configuredFile: any; private _parsedGeneratorConfiguration: any = {}; private _config: RunnerConfig; private _timeout: number; @@ -55,6 +54,7 @@ export class TestRunner extends EventEmitter { private _stdErrBuffer: (string | Buffer)[] = []; private _testResult: TestResult | null = null; private _suite: Suite; + private _loaded = false; constructor(entry: TestRunnerEntry, config: RunnerConfig, workerId: number) { super(); @@ -66,7 +66,6 @@ export class TestRunner extends EventEmitter { this._trialRun = config.trialRun; this._timeout = config.timeout; this._config = config; - this._configuredFile = entry.file + `::[${entry.configurationString}]`; for (const {name, value} of entry.configuration) this._parsedGeneratorConfiguration[name] = value; this._parsedGeneratorConfiguration['parallelIndex'] = workerId; @@ -77,14 +76,16 @@ export class TestRunner extends EventEmitter { this._trialRun = true; } - fatalError(error: Error | any) { - this._fatalError = serializeError(error); + unhandledError(error: Error | any) { if (this._testResult) { - this._testResult.error = this._fatalError; + this._testResult.error = serializeError(error); this.emit('testEnd', { id: this._testId, result: this._testResult }); + } else if (!this._loaded) { + // No current test - fatal error. + this._fatalError = serializeError(error); } this._reportDone(); } @@ -114,6 +115,7 @@ export class TestRunner extends EventEmitter { require(this._suite.file); revertBabelRequire(); this._suite._renumber(); + this._loaded = true; rerunRegistrations(this._suite.file, 'test'); await this._runSuite(this._suite); @@ -132,7 +134,6 @@ export class TestRunner extends EventEmitter { await this._runSuite(entry); else await this._runTest(entry); - } try { await this._runHooks(suite, 'afterAll', 'after'); @@ -151,6 +152,9 @@ export class TestRunner extends EventEmitter { const id = test._id; this._testId = id; + // We only know resolved skipped/flaky value in the worker, + // send it to the runner. + test._skipped = test._skipped || test.suite._isSkipped(); this.emit('testBegin', { id, skipped: test._skipped, @@ -160,13 +164,14 @@ export class TestRunner extends EventEmitter { const result: TestResult = { duration: 0, status: 'passed', + expectedStatus: test._expectedStatus, stdout: [], stderr: [], data: {} }; this._testResult = result; - if (test._skipped || test.suite._isSkipped()) { + if (test._skipped) { result.status = 'skipped'; this.emit('testEnd', { id, result }); return; @@ -181,7 +186,7 @@ export class TestRunner extends EventEmitter { await fixturePool.runTestWithFixtures(test.fn, timeout, testInfo); await this._runHooks(test.suite, 'afterEach', 'after', testInfo); } else { - result.status = 'passed'; + result.status = result.expectedStatus; } } catch (error) { // Error in the test fixture teardown. diff --git a/test-runner/src/worker.ts b/test-runner/src/worker.ts index fbe2d1479e..1d72ce72d9 100644 --- a/test-runner/src/worker.ts +++ b/test-runner/src/worker.ts @@ -49,12 +49,12 @@ let testRunner: TestRunner; process.on('unhandledRejection', (reason, promise) => { if (testRunner) - testRunner.fatalError(reason); + testRunner.unhandledError(reason); }); process.on('uncaughtException', error => { if (testRunner) - testRunner.fatalError(error); + testRunner.unhandledError(error); }); process.on('message', async message => { diff --git a/test-runner/test/assets/expected-failure.js b/test-runner/test/assets/expected-failure.js new file mode 100644 index 0000000000..6d9555f11e --- /dev/null +++ b/test-runner/test/assets/expected-failure.js @@ -0,0 +1,24 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +require('../../lib'); + +it.fail(true)('fails',() => { + expect(1 + 1).toBe(3); +}); + +it('non-empty remaining',() => { + expect(1 + 1).toBe(2); +}); diff --git a/test-runner/test/assets/nested-skip.js b/test-runner/test/assets/nested-skip.js new file mode 100644 index 0000000000..2644394274 --- /dev/null +++ b/test-runner/test/assets/nested-skip.js @@ -0,0 +1,22 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +require('../../'); + +describe.skip(true)('skipped', () => { + it('succeeds',() => { + expect(1 + 1).toBe(2); + }); +}); diff --git a/test-runner/test/assets/one-timeout.js b/test-runner/test/assets/one-timeout.js new file mode 100644 index 0000000000..2b0f4cebe5 --- /dev/null +++ b/test-runner/test/assets/one-timeout.js @@ -0,0 +1,20 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +require('../../'); + +it('timeout', async () => { + await new Promise(f => setTimeout(f, 10000)); +}); diff --git a/test-runner/test/assets/unexpected-pass.js b/test-runner/test/assets/unexpected-pass.js new file mode 100644 index 0000000000..114e650b1d --- /dev/null +++ b/test-runner/test/assets/unexpected-pass.js @@ -0,0 +1,20 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +require('../../'); + +it.fail(true)('succeeds',() => { + expect(1 + 1).toBe(2); +}); diff --git a/test-runner/test/exit-code.spec.ts b/test-runner/test/exit-code.spec.ts index bec6409fed..c9f557b544 100644 --- a/test-runner/test/exit-code.spec.ts +++ b/test-runner/test/exit-code.spec.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import colors from 'colors/safe'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; @@ -30,6 +31,14 @@ it('should fail', async () => { expect(result.failed).toBe(1); }); +it('should timeout', async () => { + const { exitCode, passed, failed, timedOut } = await runTest('one-timeout.js', { timeout: 100 }); + expect(exitCode).toBe(1); + expect(passed).toBe(0); + expect(failed).toBe(0); + expect(timedOut).toBe(1); +}); + it('should succeed', async () => { const result = await runTest('one-success.js'); expect(result.exitCode).toBe(0); @@ -85,6 +94,15 @@ it('should retry failures', async () => { expect(result.flaky).toBe(1); }); +it('should retry timeout', async () => { + const { exitCode, passed, failed, timedOut, output } = await runTest('one-timeout.js', { timeout: 100, retries: 2 }); + expect(exitCode).toBe(1); + expect(passed).toBe(0); + expect(failed).toBe(0); + expect(timedOut).toBe(1); + expect(output.split('\n')[0]).toBe(colors.red('T').repeat(3)); +}); + it('should repeat each', async () => { const { exitCode, report } = await runTest('one-success.js', { 'repeat-each': 3 }); expect(exitCode).toBe(0); @@ -106,6 +124,44 @@ it('should allow flaky', async () => { expect(result.flaky).toBe(1); }); +it('should fail on unexpected pass', async () => { + const { exitCode, failed, output } = await runTest('unexpected-pass.js'); + expect(exitCode).toBe(1); + expect(failed).toBe(1); + expect(output).toContain('passed unexpectedly'); +}); + +it('should fail on unexpected pass with retries', async () => { + const { exitCode, failed, output } = await runTest('unexpected-pass.js', { retries: 1 }); + expect(exitCode).toBe(1); + expect(failed).toBe(1); + expect(output).toContain('passed unexpectedly'); +}); + +it('should not retry unexpected pass', async () => { + const { exitCode, passed, failed, output } = await runTest('unexpected-pass.js', { retries: 2 }); + expect(exitCode).toBe(1); + expect(passed).toBe(0); + expect(failed).toBe(1); + expect(output.split('\n')[0]).toBe(colors.red('P')); +}); + +it('should not retry expected failure', async () => { + const { exitCode, passed, failed, output } = await runTest('expected-failure.js', { retries: 2 }); + expect(exitCode).toBe(0); + expect(passed).toBe(2); + expect(failed).toBe(0); + expect(output.split('\n')[0]).toBe(colors.green('f') + colors.green('·')); +}); + +it('should respect nested skip', async () => { + const { exitCode, passed, failed, skipped } = await runTest('nested-skip.js'); + expect(exitCode).toBe(0); + expect(passed).toBe(0); + expect(failed).toBe(0); + expect(skipped).toBe(1); +}); + async function runTest(filePath: string, params: any = {}) { const outputDir = path.join(__dirname, 'test-results'); const reportFile = path.join(outputDir, 'results.json'); @@ -125,14 +181,20 @@ async function runTest(filePath: string, params: any = {}) { }); const passed = (/(\d+) passed/.exec(output.toString()) || [])[1]; const failed = (/(\d+) failed/.exec(output.toString()) || [])[1]; + const timedOut = (/(\d+) timed out/.exec(output.toString()) || [])[1]; const flaky = (/(\d+) flaky/.exec(output.toString()) || [])[1]; + const skipped = (/(\d+) skipped/.exec(output.toString()) || [])[1]; const report = JSON.parse(fs.readFileSync(reportFile).toString()); + let outputStr = output.toString(); + outputStr = outputStr.substring(1, outputStr.length - 1); return { exitCode: status, - output: output.toString(), + output: outputStr, passed: parseInt(passed, 10), failed: parseInt(failed || '0', 10), + timedOut: parseInt(timedOut || '0', 10), flaky: parseInt(flaky || '0', 10), + skipped: parseInt(skipped || '0', 10), report }; } diff --git a/test/__snapshots__/page-screenshot/firefox/screenshot-webgl.png b/test/__snapshots__/page-screenshot/firefox/screenshot-webgl.png new file mode 100644 index 0000000000000000000000000000000000000000..76109512b5e2c48c3486dc154e078879925dd12e GIT binary patch literal 4016 zcmeHKYg`jo7Qd5_m@!e(sXsGXlmM5oDuxkOw4zM}g$zVssTHl!B&|?Ut0-z+O);qh zKGsJvkyou&)@O@NX|Wa77GtoF6zZ!*Y1dZKv?|q#QNW+24&NkkKpq-Gnq^vKWyDIPrG;{&+)hZ4{A(0~B(Ea@ z;wC(UJv)&9T_KTg_doe#)np6ZC$mQMPwOSnhAE@3sy8=n{%xJ5E^OC|@}~Aa|D#8F z59`p{-vp8~lVSFKjVAV+8+-0GZtQ)yXQ@*Dl}Ns&Gw|SJ&5%7qG z;?Grta8F^^$z52s%qx=5E%Tim&C+e);#jKxmr^V{HcKF$)827nJWJ1YN|l8Q?^_z4 zsU0qq*H`(H0B|KkAYOjGV^TQ`n?&;ZZNB-NVfY;iQL6%%Fe$vuTW$ZS?`>7Vu#gB@ z7#--2AcYn2$mYaX_hIw3Fp^&9s4=eKna+_4?X@KS-c+y-6Ura$@g>Eu^dm=zEVel^ za29sa2-VS!uZ97(1#u>Rg^H!^)`;CT#zdC>v;ig4-ikMP#utwww)mPeu=(34B1Mv` zSr|p83|D9?-}5ye0;3su&S2y62(sx3HdXGf3GcCU=H+TDGs0W(na;z+3FWHDz^5XX z{ymCl!iK&!9?ObW65Q|D*d0kWO_gaE>-~8lEZyt0R*%Wmr1D(#B!zZys{hbnFvg?m z=e!l$u=!S+pj2h(JF2Hj7aCCYWN(EUj7^C1VB-{6^AbsKTh()r= zED$S)MF!%dSh~Yut%mILRb7n51ZR|5t0`Gu%2g@0bmCJ?t-;ksywFm zm#oowYDWp>b8x`Zi_LT250Q;q*|q5|Sk($`z0)Us3`aFhth_SaU(&~V3N<9X+}APa zhR#zM4glUKoe0Auq1>emBv0cR1B8$^^iE#@!*4|Lvvy1MAeL6Yfjr}|@m3mD`T(0I zpKcB9<+=7XYHO#^tN2|>HWi4KmK1+UK&{kC- z8HyE~-U0V&wsJL0IxMz42j|U4)J(sq7?Vz&Kvh~3UbqeGp|%dk1Oyd!j(rWqPxLD4 z%duWSRu(k$5+``BYOhrJ11yK^TUyYfv3y)@B3n6vb?cj2L)*6DmSqN1kmVP3E6IfC zh4xF>Xo8AOuv-h7TSMuY)ZrwB_Gef>7@MagpaKa0p@X}v6C56rtJy!6tz2$EV_^9d zJ9qmGI`~(w;#MkkSV&fWYUs1Nu=&Fx;$WCpQE9{aJhAeSuF<^=o5%9jF%Z474C^PV ztun}-1mgrWBHb^#sPoJ=q4*!~KU(|nd9=3Y&tEpCX>5gJWy-XvPd=IYQt=27+XY{2 z=sESMB%M*mVF-Dc!-@ zs&tK@MmbRRt78H=RU!I$pk`$?;jIlQ=Up&H1}>HmIw{X&@AYNGP|TYke6P5QSKkom z#({CGFC$m19M5{(i}|>{towL0DLhc$djZb)BQWlRIya+upjX?wzGgi(9R@|1zq0G~ zM732vlM-LTf%tJk`xtWsMN4?5aEP~Jm{|E-P&~X9m|`Xb1tV8; zu^#RnL?SxCe}^z6uWQ6k>(*#8MGcb;h?+%}#`1BNJmiTaQv_Z`e+1EF#OBJ1nsCcG zk-QNE!GMkHst8G(ovYS(6`4+JdywU;x;9;I2$rn*)=Ym%Gi@y-fQU{@`znet=%~_v z8TzhGQCkaPOG_M<_C(-^mMXoEjrGtmCh?p(ELT&%OA4%^O5bK1Yli~Epvt?kaqcc) zn1eGXxr&1fo9n=`KX{Az1+j7&cz6uhQ6!xRnse9CXPU3J)<&?jD=Of5Nv3VHVcAUp zgp>4I02F|SIFJGGaMfus%Y<^VodcFSu!){Qsj6ziw}W~LS;-#EHQOWDX_i zckSEzxP0+FcDN>t>{pg>GOwwY9B zL_ly3n@sbDv+LHnirpumTisc>{Y4<*7>enPWa}s8YOVnT-7H_I;o}NP zp?q=%_2+cINF7ZG6%KA`QJ2uwE|3`Xlq=0IS`NKF*U2rlTWs@Lw|f?D{{&dRp&kh) zIk_I7@lpoaPO_fI0QgMjc8|sFZ-NRfT}6bzhqvv(PZGQe8USl!$c@nf!2t*% zzsS>|f^DU)%rp8~Y^u;Ms~Qen;H|_)kXc%*+|C IWYatU1^x<1#{d8T literal 0 HcmV?d00001 diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-webgl.png b/test/__snapshots__/page-screenshot/webkit/screenshot-webgl.png new file mode 100644 index 0000000000000000000000000000000000000000..18d35c5ac19eb938fce818610b271394554867a2 GIT binary patch literal 3489 zcmeHKiC0rc7tae}$)-SvgcuM+0*aC+jjX9O0tO@~ppA(jJXAucT3JM)qC}9WP=k~u zA@U)DbxjdS{VEpY17rsQ7g`h$5JMqY6QHdumGANW89hC7=FFV=&D?YD{r%?NnTsJi zIYb;4heDx6^>zaL}LDjO){vs=fu6me)Z{eEv(g`DERc0kFO4u9+Q`UC@%W#uWFr&sDzh%wl6t}8fQ)Pmy55RizD3$rDU5z ztygINLqS|IREa^zFzPwmg4dg@ANVJ8Hk^UYP`SdQh)iH)vFE|68H93 zMECW^n~}hX1G)1+Z(pD=S8V=}<~g_xuobX#~thHKumizgX-(?=E0BTD_Wa)`~ppF$Bi$ zH;gp9_zYCz;U^lGD12Qzjw*d7w?~fKl0aIy+IX&f*cOnC`@M5eyLdFV_ak-0o7bZ&6h_p0QPm*On11Vne_FLJ3)T48Z!}CD`LU49 z9QgVbC`)DzaN+pfZzj@4;z^Kcrb(1HpZ#2Bvd4yQwkus3TLO@DNNkB{{-n!4G$ zcs;YDKiuo|Qt-Givs~RYcUI&to@jWh#`us<^Vuq&WAoX{ivCipyzBOd^2PWgyTtv) zoV2(zTmKXqc9qr*dRI$@L3GINy84Pfv&I1sy4z?s%?bK*$;<`a_LGoxmm3#8vhDO0 zov(@r2#syt@fHWB>T!zusX5*%sW6TXt-7J^@vuJW! zGQCwRYIg;87tFh!%k&eVx&Ai6Mu{c2+Hi_}b%hZvUWza-Y*DU`Xa_1RfE&16dyvf|=r0O^L>&m-jt77VmuXV83Vlx1zYKSR$aGtKA_`h_fv7O)m0?`NH` z7~{KMB{OGI-rnl=Qp9m#Zn=76Jv)X92#s>7_=Je!Mx5esY6l;oM&=46+5Hg{w*r)z zLzScY0|wH~zf@-kS;tl_AYt?%ftj}%;fYmNJH=LQ_D#*SW6d=gLc%eG3C?HduyWvq zj7Lk6m_kGEHkaGAl>Q}PCzZOJC@Va>Fr9&ucz4UQ?j~St%JJcRE$qqbscQ~5cvCIa zIMAWUGPO>OZwSK3D_AbNg2N37ydKtg@Y%fr&P)MT?)buTXM94+4sq%ed-ToPtt^Dx z(?csLOHFz1{v>)BM*9K2{dVlmJuV=AiHLzDD)#PL4>9-i%;c|VTxOR|* z1M{@ah^hut@wQD~4?RiohbQrD7bpomAnRC}P#GP|>#?q#3AvPf+?PIl;)NwHar0{P z;mC+Yr;b0SnsE>DCs*^Ltq0Fx9~l0TmGz)1*hKd!LdcgMeti~Y@P!!lq5H98ePYDx z6_^KxGOJai4Q?ap?7v3W6qUJ79Bsl36wc=xDKc zA@RN4(=Ah6ad^_4U8~`dr#QvyE-$L)yU4HU(3xo(%&c7Pg9Vp#PB6xWdR19A*X~8F zw0)Eojn+7KV+iB>^xlAGMXn)XY+Hhi%*;jZs`qu7F&Acy&WWE#tg#_MrMe3+)@8*m zgqnh6Ce7{oWIZajGo#f`oQz$Pp$CZ!RYrm)a@ItlGR3j{+G6#fVO3o-FIc z95R{-sCF>EGNLT2%}!>Xx*X|p0Ik_jy2)*rl4K2Nw%IU>G^nKY&LUxzRJiF`2Zjy_ zbtToNErp&Kc_8bn|BDtuRni;ui5OQBs7lp1wN@Y63ee$$ob@Z#nelqo2{`!#__kyl1TTkZsFF1x8-ZQV>RLLyO4FmjX_bbT+I6E1zUllnnnGauuaQ<*sKWK>Q3c z(-~3l=7|_PI>Z}IeFwB;-Zuer^sR}q%=$zmQxqmMeF7Odu7GMow~Kp=xDX5CXFj@( zqpFzBq7#ty;HB9bfCPH$Xg`gP3s<5$>8$qPkJtJoWaiJ+(xJlH8Y>d`vz~jbt-JjU zMovssy1zA!3F7rkb$Wf1{H&vy3r}^8*ga}6UxL*nBasai4H;79@&gKS*hX|tlR;Ntm!h$oi#C_cOKbYE{p z0AnBqEm=5p&?L$}jUyfjAJx;NY*~;Yv0K1-yyKgZ^(2tx=*Z9~*PEdU=P$Ig)lMEr z<%~RN{kQc*mP{@l+2WccyJr~b0P9?<=Nc^0j)>{hzTRi}o5kXhI?Y_%r}4%s7|PHy zw{1t4;weoyYV&U@X;CDl0Yf;*YzxcxA<<1Cig3Z>=#p^*m)8RhozaRWvPQTtI3%on zHPwA~Er#M!9sJ{e=aBO1EPBxQa2XFm%Z5%R+dGV&M)Ig z@&tG3>&QzfRaSuHcKi0qDWtv&*uLBUlllX$WTSO9Lyp2Yhj4~UqecLqND|x*gA`D zY5!ifDxgERv|VX;7<=e&$g7D9cNiA2hQz}jYQ$lF2A0wn58j=`-!+B!&Ceg)VQhSY zqxQ`#c { await context.close(); }); -it.fail(options.CHROMIUM && !options.HEADLESS)('should fail with wrong credentials', async({browser, server}) => { +it('should fail with wrong credentials', async({browser, server}) => { server.setAuth('/empty.html', 'user', 'pass'); const context = await browser.newContext({ httpCredentials: { username: 'foo', password: 'bar' } diff --git a/test/browsercontext-locale.spec.ts b/test/browsercontext-locale.spec.ts index 14e5b80fdb..e7974c5a76 100644 --- a/test/browsercontext-locale.spec.ts +++ b/test/browsercontext-locale.spec.ts @@ -137,7 +137,7 @@ it('should be isolated between contexts', async({browser, server}) => { ]); }); -it.fail(options.FIREFOX)('should not change default locale in another context', async({browser, server}) => { +it('should not change default locale in another context', async({browser, server}) => { async function getContextLocale(context) { const page = await context.newPage(); return await page.evaluate(() => (new Intl.NumberFormat()).resolvedOptions().locale); diff --git a/test/browsercontext-page-event.spec.ts b/test/browsercontext-page-event.spec.ts index de62560d50..e8aa8c7657 100644 --- a/test/browsercontext-page-event.spec.ts +++ b/test/browsercontext-page-event.spec.ts @@ -156,7 +156,7 @@ it('should fire page lifecycle events', async function({browser, server}) { await context.close(); }); -it.fail(options.WEBKIT)('should work with Shift-clicking', async({browser, server}) => { +it.fixme(options.WEBKIT)('should work with Shift-clicking', async({browser, server}) => { // WebKit: Shift+Click does not open a new window. const context = await browser.newContext(); const page = await context.newPage(); @@ -170,7 +170,7 @@ it.fail(options.WEBKIT)('should work with Shift-clicking', async({browser, serve await context.close(); }); -it.fail(options.WEBKIT || options.FIREFOX)('should work with Ctrl-clicking', async({browser, server}) => { +it.fixme(options.WEBKIT || options.FIREFOX)('should work with Ctrl-clicking', async({browser, server}) => { // Firefox: reports an opener in this case. // WebKit: Ctrl+Click does not open a new tab. const context = await browser.newContext(); diff --git a/test/browsercontext-timezone-id.spec.ts b/test/browsercontext-timezone-id.spec.ts index 91d88f41dc..7bc4c71b8b 100644 --- a/test/browsercontext-timezone-id.spec.ts +++ b/test/browsercontext-timezone-id.spec.ts @@ -69,7 +69,7 @@ it('should work for multiple pages sharing same process', async({browser, server await context.close(); }); -it.fail(options.FIREFOX)('should not change default timezone in another context', async({browser, server}) => { +it('should not change default timezone in another context', async({browser, server}) => { async function getContextTimezone(context) { const page = await context.newPage(); return await page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone); diff --git a/test/browsertype-connect.spec.ts b/test/browsertype-connect.spec.ts index 635bc2694d..8bd117dc83 100644 --- a/test/browsertype-connect.spec.ts +++ b/test/browsertype-connect.spec.ts @@ -81,7 +81,7 @@ it.skip(options.WIRE).slow()('disconnected event should be emitted when browser expect(disconnected2).toBe(1); }); -it.skip(options.WIRE).fail(options.CHROMIUM && WIN).slow()('should handle exceptions during connect', async({browserType, remoteServer}) => { +it.skip(options.WIRE).slow()('should handle exceptions during connect', async({browserType, remoteServer}) => { const __testHookBeforeCreateBrowser = () => { throw new Error('Dummy') }; const error = await browserType.connect({ wsEndpoint: remoteServer.wsEndpoint(), __testHookBeforeCreateBrowser } as any).catch(e => e); expect(error.message).toContain('Dummy'); diff --git a/test/browsertype-launch.spec.ts b/test/browsertype-launch.spec.ts index 3e61877418..6d69b46aae 100644 --- a/test/browsertype-launch.spec.ts +++ b/test/browsertype-launch.spec.ts @@ -43,7 +43,7 @@ it.skip(options.FIREFOX)('should throw if page argument is passed', async({brows expect(waitError.message).toContain('can not specify page'); }); -it.fail(true)('should reject if launched browser fails immediately', async({browserType, defaultBrowserOptions}) => { +it.fixme(true)('should reject if launched browser fails immediately', async({browserType, defaultBrowserOptions}) => { // I'm getting ENCONRESET on this one. const options = Object.assign({}, defaultBrowserOptions, {executablePath: path.join(__dirname, 'assets', 'dummy_bad_browser_executable.js')}); let waitError = null; diff --git a/test/capabilities.spec.ts b/test/capabilities.spec.ts index a067c99b92..4a631d41b0 100644 --- a/test/capabilities.spec.ts +++ b/test/capabilities.spec.ts @@ -48,7 +48,7 @@ it('should respect CSP', async({page, server}) => { expect(await page.evaluate(() => window['testStatus'])).toBe('SUCCESS'); }); -it.fail(options.WEBKIT && (WIN || LINUX))('should play video', async({page, asset}) => { +it.fixme(options.WEBKIT && (WIN || LINUX))('should play video', async({page, asset}) => { // TODO: the test passes on Windows locally but fails on GitHub Action bot, // apparently due to a Media Pack issue in the Windows Server. // Also the test is very flaky on Linux WebKit. diff --git a/test/chromium/oopif.spec.ts b/test/chromium/oopif.spec.ts index b48be9e835..5d3467ef50 100644 --- a/test/chromium/oopif.spec.ts +++ b/test/chromium/oopif.spec.ts @@ -65,7 +65,7 @@ it.skip(!options.CHROMIUM)('should handle remote -> local -> remote transitions' expect(await countOOPIFs(browser)).toBe(1); }); -it.fail(true)('should get the proper viewport', async({browser, page, server}) => { +it.fixme(options.CHROMIUM).skip(!options.CHROMIUM)('should get the proper viewport', async({browser, page, server}) => { expect(page.viewportSize()).toEqual({width: 1280, height: 720}); await page.goto(server.PREFIX + '/dynamic-oopif.html'); expect(page.frames().length).toBe(2); diff --git a/test/click-react.spec.ts b/test/click-react.spec.ts index 4acba5a8e8..c1b345dce4 100644 --- a/test/click-react.spec.ts +++ b/test/click-react.spec.ts @@ -39,7 +39,7 @@ it.fail(true)('should report that selector does not match anymore', async ({page expect(error.message).toContain('element does not match the selector anymore'); }); -it.fail(true)('should not retarget the handle when element is recycled', async ({page, server}) => { +it.fixme(true)('should not retarget the handle when element is recycled', async ({page, server}) => { await page.goto(server.PREFIX + '/react.html'); await page.evaluate(() => { renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2', disabled: true })] )); @@ -66,7 +66,7 @@ it('should timeout when click opens alert', async({page, server}) => { await dialog.dismiss(); }); -it.fail(true)('should retarget when element is recycled during hit testing', async ({page, server}) => { +it.fixme(true)('should retarget when element is recycled during hit testing', async ({page, server}) => { await page.goto(server.PREFIX + '/react.html'); await page.evaluate(() => { renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2' })] )); @@ -81,7 +81,7 @@ it.fail(true)('should retarget when element is recycled during hit testing', asy expect(await page.evaluate('window.button2')).toBe(undefined); }); -it.fail(true)('should retarget when element is recycled before enabled check', async ({page, server}) => { +it.fixme(true)('should retarget when element is recycled before enabled check', async ({page, server}) => { await page.goto(server.PREFIX + '/react.html'); await page.evaluate(() => { renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2', disabled: true })] )); diff --git a/test/click.spec.ts b/test/click.spec.ts index 61b4cac874..26eed718c6 100644 --- a/test/click.spec.ts +++ b/test/click.spec.ts @@ -322,7 +322,7 @@ it('should click the button inside an iframe', async({page, server}) => { expect(await frame.evaluate(() => window['result'])).toBe('Clicked'); }); -it.fail(options.CHROMIUM || options.WEBKIT)('should click the button with fixed position inside an iframe', async({page, server}) => { +it.fixme(options.CHROMIUM || options.WEBKIT)('should click the button with fixed position inside an iframe', async({page, server}) => { // @see https://github.com/GoogleChrome/puppeteer/issues/4110 // @see https://bugs.chromium.org/p/chromium/issues/detail?id=986390 // @see https://chromium-review.googlesource.com/c/chromium/src/+/1742784 diff --git a/test/coverage.js b/test/coverage.js index 18eb40f63e..86cb4e9cb3 100644 --- a/test/coverage.js +++ b/test/coverage.js @@ -24,12 +24,12 @@ function traceAPICoverage(apiCoverage, api, events) { const uninstalls = []; for (const [name, classType] of Object.entries(api)) { - // console.log('trace', name); const className = name.substring(0, 1).toLowerCase() + name.substring(1); for (const methodName of Reflect.ownKeys(classType.prototype)) { const method = Reflect.get(classType.prototype, methodName); if (methodName === 'constructor' || typeof methodName !== 'string' || methodName.startsWith('_') || typeof method !== 'function') continue; + apiCoverage.set(`${className}.${methodName}`, false); const override = function(...args) { apiCoverage.set(`${className}.${methodName}`, true); diff --git a/test/defaultbrowsercontext-2.spec.ts b/test/defaultbrowsercontext-2.spec.ts index b2636ed36b..a7d709536e 100644 --- a/test/defaultbrowsercontext-2.spec.ts +++ b/test/defaultbrowsercontext-2.spec.ts @@ -75,7 +75,7 @@ it('should support extraHTTPHeaders option', async ({server, launchPersistent}) expect(request.headers['foo']).toBe('bar'); }); -it('should accept userDataDir', async ({launchPersistent, tmpDir}) => { +it.flaky(options.CHROMIUM)('should accept userDataDir', async ({launchPersistent, tmpDir}) => { const {page, context} = await launchPersistent(); // Note: we need an open page to make sure its functional. expect(fs.readdirSync(tmpDir).length).toBeGreaterThan(0); @@ -111,7 +111,7 @@ it.slow()('should restore state from userDataDir', async({browserType, defaultBr await removeUserDataDir(userDataDir2); }); -it.fail(options.CHROMIUM && (WIN || MAC)).slow()('should restore cookies from userDataDir', async({browserType, defaultBrowserOptions, server, launchPersistent}) => { +it.slow()('should restore cookies from userDataDir', async({browserType, defaultBrowserOptions, server, launchPersistent}) => { const userDataDir = await makeUserDataDir(); const browserContext = await browserType.launchPersistentContext(userDataDir, defaultBrowserOptions); const page = await browserContext.newPage(); diff --git a/test/dialog.spec.ts b/test/dialog.spec.ts index 1bd6ece5c2..d5e18e2afd 100644 --- a/test/dialog.spec.ts +++ b/test/dialog.spec.ts @@ -62,7 +62,7 @@ it('should dismiss the confirm prompt', async({page}) => { expect(result).toBe(false); }); -it.fail(options.WEBKIT)('should be able to close context with open alert', async({browser}) => { +it.fixme(options.WEBKIT && MAC)('should be able to close context with open alert', async({browser}) => { const context = await browser.newContext(); const page = await context.newPage(); const alertPromise = page.waitForEvent('dialog'); diff --git a/test/download.spec.ts b/test/download.spec.ts index 332c48728b..4798c79c9f 100644 --- a/test/download.spec.ts +++ b/test/download.spec.ts @@ -250,7 +250,7 @@ it(`should report download path within page.on('download', …) handler for Blob expect(fs.readFileSync(path).toString()).toBe('Hello world'); await page.close(); }) -it.fail(options.FIREFOX || options.WEBKIT)('should report alt-click downloads', async({browser, server}) => { +it.fixme(options.FIREFOX || options.WEBKIT)('should report alt-click downloads', async({browser, server}) => { // Firefox does not download on alt-click by default. // Our WebKit embedder does not download on alt-click, although Safari does. server.setRoute('/download', (req, res) => { @@ -271,7 +271,7 @@ it.fail(options.FIREFOX || options.WEBKIT)('should report alt-click downloads', await page.close(); }); -it.fail(options.CHROMIUM && !options.HEADLESS)('should report new window downloads', async({browser, server}) => { +it.fixme(options.CHROMIUM && !options.HEADLESS)('should report new window downloads', async({browser, server}) => { // TODO: - the test fails in headful Chromium as the popup page gets closed along // with the session before download completed event arrives. // - WebKit doesn't close the popup page diff --git a/test/elementhandle-owner-frame.spec.ts b/test/elementhandle-owner-frame.spec.ts index d81352eff0..7d4ae67148 100644 --- a/test/elementhandle-owner-frame.spec.ts +++ b/test/elementhandle-owner-frame.spec.ts @@ -17,6 +17,7 @@ import './playwright.fixtures'; import utils from './utils'; +import { options } from './playwright.fixtures'; it('should work', async ({ page, server }) => { await page.goto(server.EMPTY_PAGE); @@ -34,7 +35,7 @@ it('should work for cross-process iframes', async ({ page, server }) => { expect(await elementHandle.ownerFrame()).toBe(frame); }); -it('should work for document', async ({ page, server }) => { +it.flaky(WIN && options.WEBKIT)('should work for document', async ({ page, server }) => { await page.goto(server.EMPTY_PAGE); await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); const frame = page.frames()[1]; diff --git a/test/elementhandle-screenshot.spec.ts b/test/elementhandle-screenshot.spec.ts index 65d3861c4b..90659839e5 100644 --- a/test/elementhandle-screenshot.spec.ts +++ b/test/elementhandle-screenshot.spec.ts @@ -351,7 +351,7 @@ it.skip(ffheadful || options.WIRE)('should restore viewport after element screen await context.close(); }); -it.skip(ffheadful)('should wait for element to stop moving', async({page, server, golden}) => { +it.skip(ffheadful).flaky(options.WEBKIT && !options.HEADLESS && LINUX)('should wait for element to stop moving', async ({ page, server, golden }) => { await page.setViewportSize({ width: 500, height: 500 }); await page.goto(server.PREFIX + '/grid.html'); const elementHandle = await page.$('.box:nth-of-type(3)'); diff --git a/test/elementhandle-wait-for-element-state.spec.ts b/test/elementhandle-wait-for-element-state.spec.ts index 440ecbdf8b..1c860f8493 100644 --- a/test/elementhandle-wait-for-element-state.spec.ts +++ b/test/elementhandle-wait-for-element-state.spec.ts @@ -16,6 +16,7 @@ */ import './playwright.fixtures'; +import { options } from './playwright.fixtures'; async function giveItAChanceToResolve(page) { for (let i = 0; i < 5; i++) diff --git a/test/frame-evaluate.spec.ts b/test/frame-evaluate.spec.ts index 5d9e066401..2ca0423bbb 100644 --- a/test/frame-evaluate.spec.ts +++ b/test/frame-evaluate.spec.ts @@ -147,7 +147,7 @@ it.fail(options.CHROMIUM || options.FIREFOX)('should work in iframes that failed expect(await page.frames()[1].$('div')).toBeTruthy(); }); -it.fail(options.CHROMIUM)('should work in iframes that interrupted initial javascript url navigation', async({page, server}) => { +it.fixme(options.CHROMIUM)('should work in iframes that interrupted initial javascript url navigation', async({page, server}) => { // Chromium does not report isolated world for the iframe. await page.goto(server.EMPTY_PAGE); await page.evaluate(() => { diff --git a/test/frame-hierarchy.spec.ts b/test/frame-hierarchy.spec.ts index ec36d239d4..29ba88eda6 100644 --- a/test/frame-hierarchy.spec.ts +++ b/test/frame-hierarchy.spec.ts @@ -171,7 +171,7 @@ it('should report different frame instance when frame re-attaches', async({page, expect(frame1).not.toBe(frame2); }); -it.fail(options.FIREFOX)('should refuse to display x-frame-options:deny iframe', async({page, server}) => { +it.fixme(options.FIREFOX)('should refuse to display x-frame-options:deny iframe', async({page, server}) => { server.setRoute('/x-frame-options-deny.html', async (req, res) => { res.setHeader('Content-Type', 'text/html'); res.setHeader('X-Frame-Options', 'DENY'); diff --git a/test/headful.spec.ts b/test/headful.spec.ts index 0a3221b2f8..39ecc6f2ec 100644 --- a/test/headful.spec.ts +++ b/test/headful.spec.ts @@ -128,7 +128,7 @@ it('should(not) block third party cookies', async({browserType, defaultBrowserOp await browser.close(); }); -it.fail(options.WEBKIT)('should not override viewport size when passed null', async function({browserType, defaultBrowserOptions, server}) { +it.fixme(options.WEBKIT)('should not override viewport size when passed null', async function({browserType, defaultBrowserOptions, server}) { // Our WebKit embedder does not respect window features. const browser = await browserType.launch({...defaultBrowserOptions, headless: false }); const context = await browser.newContext({ viewport: null }); diff --git a/test/mouse.spec.ts b/test/mouse.spec.ts index 565e983c7a..e242054f73 100644 --- a/test/mouse.spec.ts +++ b/test/mouse.spec.ts @@ -27,7 +27,7 @@ function dimensions() { }; } -it.fail(options.FIREFOX && WIN)('should click the document', async({page, server}) => { +it.flaky(options.FIREFOX && WIN)('should click the document', async({page, server}) => { // Occasionally times out on options.FIREFOX on Windows: https://github.com/microsoft/playwright/pull/1911/checks?check_run_id=607149016 await page.evaluate(() => { window["clickPromise"] = new Promise(resolve => { diff --git a/test/page-add-script-tag.spec.ts b/test/page-add-script-tag.spec.ts index 6204867d19..99636444f1 100644 --- a/test/page-add-script-tag.spec.ts +++ b/test/page-add-script-tag.spec.ts @@ -88,7 +88,7 @@ it('should work with content', async({page, server}) => { expect(await page.evaluate(() => window['__injected'])).toBe(35); }); -it.fail(options.FIREFOX)('should throw when added with content to the CSP page', async({page, server}) => { +it('should throw when added with content to the CSP page', async({page, server}) => { // Firefox fires onload for blocked script before it issues the CSP console error. await page.goto(server.PREFIX + '/csp.html'); let error = null; diff --git a/test/page-emulate-media.spec.ts b/test/page-emulate-media.spec.ts index fcb4d9cfdf..0a4660727a 100644 --- a/test/page-emulate-media.spec.ts +++ b/test/page-emulate-media.spec.ts @@ -114,7 +114,7 @@ it('should work in cross-process iframe', async({browser, server}) => { await page.close(); }); -it.fail(options.FIREFOX)('should change the actual colors in css', async({page}) => { +it('should change the actual colors in css', async({page}) => { await page.setContent(`