From ba06fb2f0d905c8b56c36d8ae14c98260e8b907a Mon Sep 17 00:00:00 2001 From: Pavel Date: Mon, 2 Mar 2020 14:57:09 -0800 Subject: [PATCH] test: mark some tests as skipped --- test/accessibility.spec.js | 6 +-- test/navigation.spec.js | 2 +- test/page.spec.js | 2 +- test/playwright.spec.js | 94 ++++++++++++++++++---------------- test/test.js | 2 +- test/web.spec.js | 2 +- utils/testrunner/Reporter.js | 32 +++++++----- utils/testrunner/TestRunner.js | 42 ++++++++++----- 8 files changed, 103 insertions(+), 79 deletions(-) diff --git a/test/accessibility.spec.js b/test/accessibility.spec.js index b399b81a33..2bfa8f7cc4 100644 --- a/test/accessibility.spec.js +++ b/test/accessibility.spec.js @@ -140,7 +140,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT, expect(await page.accessibility.snapshot()).toEqual(golden); }); // WebKit rich text accessibility is iffy - !WEBKIT && it('rich text editable fields should have children', async function({page}) { + it.skip(WEBKIT)('rich text editable fields should have children', async function({page}) { await page.setContent(`
Edit this image: my fake image @@ -171,7 +171,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT, expect(snapshot.children[0]).toEqual(golden); }); // WebKit rich text accessibility is iffy - !WEBKIT && it('rich text editable fields with role should have children', async function({page}) { + it.skip(WEBKIT)('rich text editable fields with role should have children', async function({page}) { await page.setContent(`
Edit this image: my fake image @@ -201,7 +201,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT, }); // Firefox does not support contenteditable="plaintext-only". // WebKit rich text accessibility is iffy - !FFOX && !WEBKIT && describe('plaintext contenteditable', function() { + describe.skip(FFOX || WEBKIT)('plaintext contenteditable', function() { it('plain text field with role should not have children', async function({page}) { await page.setContent(`
Edit this image:my fake image
`); diff --git a/test/navigation.spec.js b/test/navigation.spec.js index 5bf278171a..b54b45e1f9 100644 --- a/test/navigation.spec.js +++ b/test/navigation.spec.js @@ -121,7 +121,7 @@ module.exports.describe = function({testRunner, expect, playwright, MAC, WIN, FF }); await page.goto(server.PREFIX + '/frames/one-frame.html'); }); - !WEBKIT && it('should fail when server returns 204', async({page, server}) => { + it.fail(WEBKIT)('should fail when server returns 204', async({page, server}) => { // Webkit just loads an empty page. server.setRoute('/empty.html', (req, res) => { res.statusCode = 204; diff --git a/test/page.spec.js b/test/page.spec.js index bc359adc23..a36c5ecb16 100644 --- a/test/page.spec.js +++ b/test/page.spec.js @@ -652,7 +652,7 @@ module.exports.describe = function({testRunner, expect, headless, playwright, FF expect(await page.evaluate(() => __injected)).toBe(42); }); - (CHROMIUM || FFOX) && it('should include sourceURL when path is provided', async({page, server}) => { + it.skip(WEBKIT)('should include sourceURL when path is provided', async({page, server}) => { await page.goto(server.EMPTY_PAGE); await page.addScriptTag({ path: path.join(__dirname, 'assets/injectedfile.js') }); const result = await page.evaluate(() => __injectedError.stack); diff --git a/test/playwright.spec.js b/test/playwright.spec.js index 2caacea451..94e684e685 100644 --- a/test/playwright.spec.js +++ b/test/playwright.spec.js @@ -99,7 +99,7 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => { ASSETS_DIR, }; - describe('Browser', function() { + describe('', function() { beforeAll(async state => { state.browser = await playwright.launch(defaultBrowserOptions); state.browserServer = state.browser.__server__; @@ -142,7 +142,7 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => { await state.tearDown(); }); - describe('Page', function() { + describe('', function() { beforeEach(async state => { state.context = await state.browser.newContext(); state.page = await state.context.newPage(); @@ -156,66 +156,70 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => { // Page-level tests that are given a browser, a context and a page. // Each test is launched in a new browser context. - testRunner.loadTests(require('./accessibility.spec.js'), testOptions); - testRunner.loadTests(require('./click.spec.js'), testOptions); - testRunner.loadTests(require('./cookies.spec.js'), testOptions); - testRunner.loadTests(require('./dialog.spec.js'), testOptions); - testRunner.loadTests(require('./elementhandle.spec.js'), testOptions); - testRunner.loadTests(require('./emulation.spec.js'), testOptions); - testRunner.loadTests(require('./evaluation.spec.js'), testOptions); - testRunner.loadTests(require('./frame.spec.js'), testOptions); - testRunner.loadTests(require('./focus.spec.js'), testOptions); - testRunner.loadTests(require('./input.spec.js'), testOptions); - testRunner.loadTests(require('./jshandle.spec.js'), testOptions); - testRunner.loadTests(require('./keyboard.spec.js'), testOptions); - testRunner.loadTests(require('./mouse.spec.js'), testOptions); - testRunner.loadTests(require('./navigation.spec.js'), testOptions); - testRunner.loadTests(require('./network.spec.js'), testOptions); - testRunner.loadTests(require('./page.spec.js'), testOptions); - testRunner.loadTests(require('./queryselector.spec.js'), testOptions); - testRunner.loadTests(require('./screenshot.spec.js'), testOptions); - testRunner.loadTests(require('./waittask.spec.js'), testOptions); - testRunner.loadTests(require('./interception.spec.js'), testOptions); - testRunner.loadTests(require('./geolocation.spec.js'), testOptions); - testRunner.loadTests(require('./workers.spec.js'), testOptions); - testRunner.loadTests(require('./capabilities.spec.js'), testOptions); + describe('[Accessibility]', () => testRunner.loadTests(require('./accessibility.spec.js'), testOptions)); + describe('[Driver]', () => { + testRunner.loadTests(require('./click.spec.js'), testOptions); + testRunner.loadTests(require('./cookies.spec.js'), testOptions); + testRunner.loadTests(require('./dialog.spec.js'), testOptions); + testRunner.loadTests(require('./elementhandle.spec.js'), testOptions); + testRunner.loadTests(require('./emulation.spec.js'), testOptions); + testRunner.loadTests(require('./evaluation.spec.js'), testOptions); + testRunner.loadTests(require('./frame.spec.js'), testOptions); + testRunner.loadTests(require('./focus.spec.js'), testOptions); + testRunner.loadTests(require('./input.spec.js'), testOptions); + testRunner.loadTests(require('./jshandle.spec.js'), testOptions); + testRunner.loadTests(require('./keyboard.spec.js'), testOptions); + testRunner.loadTests(require('./mouse.spec.js'), testOptions); + testRunner.loadTests(require('./navigation.spec.js'), testOptions); + testRunner.loadTests(require('./network.spec.js'), testOptions); + testRunner.loadTests(require('./page.spec.js'), testOptions); + testRunner.loadTests(require('./queryselector.spec.js'), testOptions); + testRunner.loadTests(require('./screenshot.spec.js'), testOptions); + testRunner.loadTests(require('./waittask.spec.js'), testOptions); + testRunner.loadTests(require('./interception.spec.js'), testOptions); + testRunner.loadTests(require('./geolocation.spec.js'), testOptions); + testRunner.loadTests(require('./workers.spec.js'), testOptions); + testRunner.loadTests(require('./capabilities.spec.js'), testOptions); + }); - if (CHROMIUM) { + describe.skip(!CHROMIUM)('[Chromium]', () => { testRunner.loadTests(require('./chromium/chromium.spec.js'), testOptions); testRunner.loadTests(require('./chromium/coverage.spec.js'), testOptions); testRunner.loadTests(require('./chromium/pdf.spec.js'), testOptions); testRunner.loadTests(require('./chromium/session.spec.js'), testOptions); - } + }); - if (CHROMIUM || FFOX) { + describe('[Permissions]', () => { testRunner.loadTests(require('./features/permissions.spec.js'), testOptions); - } - - if (WEBKIT) { - testRunner.loadTests(require('./webkit/provisional.spec.js'), testOptions); - } + }); }); // Browser-level tests that are given a browser. - testRunner.loadTests(require('./browser.spec.js'), testOptions); - testRunner.loadTests(require('./browsercontext.spec.js'), testOptions); - testRunner.loadTests(require('./ignorehttpserrors.spec.js'), testOptions); - testRunner.loadTests(require('./popup.spec.js'), testOptions); + describe('[Driver]', () => { + testRunner.loadTests(require('./browser.spec.js'), testOptions); + testRunner.loadTests(require('./browsercontext.spec.js'), testOptions); + testRunner.loadTests(require('./ignorehttpserrors.spec.js'), testOptions); + testRunner.loadTests(require('./popup.spec.js'), testOptions); + }); }); // Top-level tests that launch Browser themselves. - testRunner.loadTests(require('./defaultbrowsercontext.spec.js'), testOptions); - testRunner.loadTests(require('./fixtures.spec.js'), testOptions); - testRunner.loadTests(require('./launcher.spec.js'), testOptions); - testRunner.loadTests(require('./headful.spec.js'), testOptions); - testRunner.loadTests(require('./multiclient.spec.js'), testOptions); + describe('[Driver]', () => { + testRunner.loadTests(require('./defaultbrowsercontext.spec.js'), testOptions); + testRunner.loadTests(require('./fixtures.spec.js'), testOptions); + testRunner.loadTests(require('./launcher.spec.js'), testOptions); + testRunner.loadTests(require('./headful.spec.js'), testOptions); + testRunner.loadTests(require('./multiclient.spec.js'), testOptions); + }); - if (CHROMIUM) { + describe.skip(!CHROMIUM)('[Chromium]', () => { testRunner.loadTests(require('./chromium/launcher.spec.js'), testOptions); testRunner.loadTests(require('./chromium/headful.spec.js'), testOptions); testRunner.loadTests(require('./chromium/oopif.spec.js'), testOptions); testRunner.loadTests(require('./chromium/tracing.spec.js'), testOptions); - } + }); - testRunner.loadTests(require('./web.spec.js'), testOptions); + describe('[Driver]', () => { + testRunner.loadTests(require('./web.spec.js'), testOptions); + }); }; diff --git a/test/test.js b/test/test.js index 7d5f63c431..9301130654 100644 --- a/test/test.js +++ b/test/test.js @@ -125,7 +125,7 @@ new Reporter(testRunner, { verbose: process.argv.includes('--verbose'), summary: !process.argv.includes('--verbose'), showSlowTests: process.env.CI ? 5 : 0, - showSkippedTests: 10, + showMarkedAsFailingTests: 10, }); // await utils.initializeFlakinessDashboardIfNeeded(testRunner); diff --git a/test/web.spec.js b/test/web.spec.js index d18d2b3cb7..997440c144 100644 --- a/test/web.spec.js +++ b/test/web.spec.js @@ -22,7 +22,7 @@ module.exports.describe = function({testRunner, expect, defaultBrowserOptions, p const {it, fit, xit, dit} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; - (CHROMIUM || FFOX) && describe('Web SDK', function() { + describe('Web SDK', function() { beforeAll(async state => { state.controlledBrowserApp = await playwright.launchServer(defaultBrowserOptions); state.hostBrowser = await playwright.launch(defaultBrowserOptions); diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js index f40b7b6ee8..2be1036d98 100644 --- a/utils/testrunner/Reporter.js +++ b/utils/testrunner/Reporter.js @@ -22,14 +22,14 @@ class Reporter { constructor(runner, options = {}) { const { showSlowTests = 3, - showSkippedTests = Infinity, + showMarkedAsFailingTests = Infinity, verbose = false, summary = true, } = options; this._filePathToLines = new Map(); this._runner = runner; this._showSlowTests = showSlowTests; - this._showSkippedTests = showSkippedTests; + this._showMarkedAsFailingTests = showMarkedAsFailingTests; this._verbose = verbose; this._summary = summary; this._testCounter = 0; @@ -110,16 +110,17 @@ class Reporter { } const skippedTests = this._runner.skippedTests(); - if (this._showSkippedTests && this._summary && skippedTests.length) { - if (skippedTests.length > 0) { - console.log('\nSkipped:'); - skippedTests.slice(0, this._showSkippedTests).forEach((test, index) => { + const markedAsFailingTests = this._runner.markedAsFailingTests(); + if (this._showMarkedAsFailingTests && this._summary && markedAsFailingTests.length) { + if (markedAsFailingTests.length > 0) { + console.log('\nMarked as failing:'); + markedAsFailingTests.slice(0, this._showMarkedAsFailingTests).forEach((test, index) => { console.log(`${index + 1}) ${test.fullName} (${formatLocation(test.location)})`); }); } - if (this._showSkippedTests < skippedTests.length) { + if (this._showMarkedAsFailingTests < markedAsFailingTests.length) { console.log(''); - console.log(`... and ${colors.yellow(skippedTests.length - this._showSkippedTests)} more skipped tests ...`); + console.log(`... and ${colors.yellow(markedAsFailingTests.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`); } } @@ -139,12 +140,14 @@ class Reporter { const tests = this._runner.tests(); const executedTests = tests.filter(test => test.result); - const okTestsLength = executedTests.length - failedTests.length - skippedTests.length; + const okTestsLength = executedTests.length - failedTests.length - markedAsFailingTests.length - skippedTests.length; let summaryText = ''; - if (failedTests.length || skippedTests.length) { + if (failedTests.length || markedAsFailingTests.length) { const summary = [`ok - ${colors.green(okTestsLength)}`]; if (failedTests.length) summary.push(`failed - ${colors.red(failedTests.length)}`); + if (markedAsFailingTests.length) + summary.push(`marked as failing - ${colors.yellow(markedAsFailingTests.length)}`); if (skippedTests.length) summary.push(`skipped - ${colors.yellow(skippedTests.length)}`); summaryText = ` (${summary.join(', ')})`; @@ -167,9 +170,11 @@ class Reporter { this._printVerboseTestResult(this._testCounter, test, workerId); } else { if (test.result === 'ok') - process.stdout.write(colors.green('.')); + process.stdout.write(colors.green('\u00B7')); else if (test.result === 'skipped') - process.stdout.write(colors.yellow('*')); + process.stdout.write(colors.yellow('\u00B7')); + else if (test.result === 'markedAsFailing') + process.stdout.write(colors.yellow('\u00D7')); else if (test.result === 'failed') process.stdout.write(colors.red('F')); else if (test.result === 'crashed') @@ -192,7 +197,8 @@ class Reporter { } else if (test.result === 'crashed') { console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName} (${formatLocation(test.location)})`); } else if (test.result === 'skipped') { - console.log(`${prefix} ${colors.yellow('[SKIP]')} ${test.fullName} (${formatLocation(test.location)})`); + } else if (test.result === 'markedAsFailing') { + console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName} (${formatLocation(test.location)})`); } else if (test.result === 'timedout') { console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout}ms]`)} ${test.fullName} (${formatLocation(test.location)})`); } else if (test.result === 'failed') { diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index 02e9df4e61..a71e57d3d6 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -72,13 +72,14 @@ const TestMode = { Run: 'run', Skip: 'skip', Focus: 'focus', - ExpectToFail: 'fail', + MarkAsFailing: 'markAsFailing', Flake: 'flake' }; const TestResult = { Ok: 'ok', - ExpectToFail: 'skipped', // User marked as failed + MarkedAsFailing: 'markedAsFailing', // User marked as failed + Skipped: 'skipped', // User marked as skipped Failed: 'failed', // Exception happened during running TimedOut: 'timedout', // Timeout Exceeded while running Terminated: 'terminated', // Execution terminated @@ -138,7 +139,7 @@ class TestPass { for (let suite = test.suite; suite; suite = suite.parentSuite) this._workerDistribution.set(suite, workerId); // Do not shard skipped tests across workers. - if (test.declaredMode !== TestMode.ExpectToFail) + if (test.declaredMode !== TestMode.MarkAsFailing && test.declaredMode !== TestMode.Skip) workerId = (workerId + 1) % parallel; } @@ -200,8 +201,13 @@ class TestPass { if (this._termination) return; this._runner._willStartTest(test, workerId); - if (test.declaredMode === TestMode.ExpectToFail) { - test.result = TestResult.ExpectToFail; + if (test.declaredMode === TestMode.MarkAsFailing) { + test.result = TestResult.MarkedAsFailing; + this._runner._didFinishTest(test, workerId); + return; + } + if (test.declaredMode === TestMode.Skip) { + test.result = TestResult.Skipped; this._runner._didFinishTest(test, workerId); return; } @@ -296,7 +302,7 @@ function specBuilder(action) { }; func.fail = condition => { if (condition) - mode = TestMode.ExpectToFail; + mode = TestMode.MarkAsFailing; return func; }; func.flake = condition => { @@ -368,14 +374,20 @@ class TestRunner extends EventEmitter { } _addTest(name, callback, mode, timeout) { - if (mode === TestMode.Skip) - return; let suite = this._currentSuite; - let expectToFail = suite.declaredMode === TestMode.ExpectToFail; + let markedAsFailing = suite.declaredMode === TestMode.MarkAsFailing; while ((suite = suite.parentSuite)) - expectToFail |= suite.declaredMode === TestMode.ExpectToFail; - if (expectToFail) - mode = TestMode.ExpectToFail; + markedAsFailing |= suite.declaredMode === TestMode.MarkAsFailing; + if (markedAsFailing) + mode = TestMode.MarkAsFailing; + + suite = this._currentSuite; + let skip = suite.declaredMode === TestMode.Skip; + while ((suite = suite.parentSuite)) + skip |= suite.declaredMode === TestMode.Skip; + if (skip) + mode = TestMode.Skip; + const test = new Test(this._currentSuite, name, callback, mode, timeout); this._currentSuite.children.push(test); this._tests.push(test); @@ -384,8 +396,6 @@ class TestRunner extends EventEmitter { } _addSuite(mode, name, callback, ...args) { - if (mode === TestMode.Skip) - return; const oldSuite = this._currentSuite; const suite = new Suite(this._currentSuite, name, mode); this._currentSuite.children.push(suite); @@ -491,6 +501,10 @@ class TestRunner extends EventEmitter { return this._tests.filter(test => test.result === 'skipped'); } + markedAsFailingTests() { + return this._tests.filter(test => test.result === 'markedAsFailing'); + } + parallel() { return this._parallel; }