Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

test: mark some tests as skipped

Browse source
This commit is contained in:
Pavel 2020-03-02 14:57:09 -08:00
parent 1186998bd8
commit ba06fb2f0d
8 changed files with 103 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
test/accessibility.spec.js
View file

@ -140,7 +140,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT,
expect(await page.accessibility.snapshot()).toEqual(golden); expect(await page.accessibility.snapshot()).toEqual(golden);
}); });
// WebKit rich text accessibility is iffy // WebKit rich text accessibility is iffy
!WEBKIT && it('rich text editable fields should have children', async function({page}) { it.skip(WEBKIT)('rich text editable fields should have children', async function({page}) {
await page.setContent(` await page.setContent(`
<div contenteditable="true"> <div contenteditable="true">
Edit this image: <img src="fakeimage.png" alt="my fake image"> Edit this image: <img src="fakeimage.png" alt="my fake image">
@ -171,7 +171,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT,
expect(snapshot.children[0]).toEqual(golden); expect(snapshot.children[0]).toEqual(golden);
}); });
// WebKit rich text accessibility is iffy // WebKit rich text accessibility is iffy
!WEBKIT && it('rich text editable fields with role should have children', async function({page}) { it.skip(WEBKIT)('rich text editable fields with role should have children', async function({page}) {
await page.setContent(` await page.setContent(`
<div contenteditable="true" role='textbox'> <div contenteditable="true" role='textbox'>
Edit this image: <img src="fakeimage.png" alt="my fake image"> Edit this image: <img src="fakeimage.png" alt="my fake image">
@ -201,7 +201,7 @@ module.exports.describe = function({testRunner, expect, FFOX, CHROMIUM, WEBKIT,
}); });
// Firefox does not support contenteditable="plaintext-only". // Firefox does not support contenteditable="plaintext-only".
// WebKit rich text accessibility is iffy // WebKit rich text accessibility is iffy
!FFOX && !WEBKIT && describe('plaintext contenteditable', function() { describe.skip(FFOX || WEBKIT)('plaintext contenteditable', function() {
it('plain text field with role should not have children', async function({page}) { it('plain text field with role should not have children', async function({page}) {
await page.setContent(` await page.setContent(`
<div contenteditable="plaintext-only" role='textbox'>Edit this image:<img src="fakeimage.png" alt="my fake image"></div>`); <div contenteditable="plaintext-only" role='textbox'>Edit this image:<img src="fakeimage.png" alt="my fake image"></div>`);

2
test/navigation.spec.js
View file

@ -121,7 +121,7 @@ module.exports.describe = function({testRunner, expect, playwright, MAC, WIN, FF
}); });
await page.goto(server.PREFIX + '/frames/one-frame.html'); await page.goto(server.PREFIX + '/frames/one-frame.html');
}); });
!WEBKIT && it('should fail when server returns 204', async({page, server}) => { it.fail(WEBKIT)('should fail when server returns 204', async({page, server}) => {
// Webkit just loads an empty page. // Webkit just loads an empty page.
server.setRoute('/empty.html', (req, res) => { server.setRoute('/empty.html', (req, res) => {
res.statusCode = 204; res.statusCode = 204;

2
test/page.spec.js
View file

@ -652,7 +652,7 @@ module.exports.describe = function({testRunner, expect, headless, playwright, FF
expect(await page.evaluate(() => __injected)).toBe(42); expect(await page.evaluate(() => __injected)).toBe(42);
}); });
(CHROMIUM || FFOX) && it('should include sourceURL when path is provided', async({page, server}) => { it.skip(WEBKIT)('should include sourceURL when path is provided', async({page, server}) => {
await page.goto(server.EMPTY_PAGE); await page.goto(server.EMPTY_PAGE);
await page.addScriptTag({ path: path.join(__dirname, 'assets/injectedfile.js') }); await page.addScriptTag({ path: path.join(__dirname, 'assets/injectedfile.js') });
const result = await page.evaluate(() => __injectedError.stack); const result = await page.evaluate(() => __injectedError.stack);

94
test/playwright.spec.js
View file

@ -99,7 +99,7 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {
ASSETS_DIR, ASSETS_DIR,
}; };
describe('Browser', function() { describe('', function() {
beforeAll(async state => { beforeAll(async state => {
state.browser = await playwright.launch(defaultBrowserOptions); state.browser = await playwright.launch(defaultBrowserOptions);
state.browserServer = state.browser.__server__; state.browserServer = state.browser.__server__;
@ -142,7 +142,7 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {
await state.tearDown(); await state.tearDown();
}); });
describe('Page', function() { describe('', function() {
beforeEach(async state => { beforeEach(async state => {
state.context = await state.browser.newContext(); state.context = await state.browser.newContext();
state.page = await state.context.newPage(); state.page = await state.context.newPage();
@ -156,66 +156,70 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => {
// Page-level tests that are given a browser, a context and a page. // Page-level tests that are given a browser, a context and a page.
// Each test is launched in a new browser context. // Each test is launched in a new browser context.
testRunner.loadTests(require('./accessibility.spec.js'), testOptions); describe('[Accessibility]', () => testRunner.loadTests(require('./accessibility.spec.js'), testOptions));
testRunner.loadTests(require('./click.spec.js'), testOptions); describe('[Driver]', () => {
testRunner.loadTests(require('./cookies.spec.js'), testOptions); testRunner.loadTests(require('./click.spec.js'), testOptions);
testRunner.loadTests(require('./dialog.spec.js'), testOptions); testRunner.loadTests(require('./cookies.spec.js'), testOptions);
testRunner.loadTests(require('./elementhandle.spec.js'), testOptions); testRunner.loadTests(require('./dialog.spec.js'), testOptions);
testRunner.loadTests(require('./emulation.spec.js'), testOptions); testRunner.loadTests(require('./elementhandle.spec.js'), testOptions);
testRunner.loadTests(require('./evaluation.spec.js'), testOptions); testRunner.loadTests(require('./emulation.spec.js'), testOptions);
testRunner.loadTests(require('./frame.spec.js'), testOptions); testRunner.loadTests(require('./evaluation.spec.js'), testOptions);
testRunner.loadTests(require('./focus.spec.js'), testOptions); testRunner.loadTests(require('./frame.spec.js'), testOptions);
testRunner.loadTests(require('./input.spec.js'), testOptions); testRunner.loadTests(require('./focus.spec.js'), testOptions);
testRunner.loadTests(require('./jshandle.spec.js'), testOptions); testRunner.loadTests(require('./input.spec.js'), testOptions);
testRunner.loadTests(require('./keyboard.spec.js'), testOptions); testRunner.loadTests(require('./jshandle.spec.js'), testOptions);
testRunner.loadTests(require('./mouse.spec.js'), testOptions); testRunner.loadTests(require('./keyboard.spec.js'), testOptions);
testRunner.loadTests(require('./navigation.spec.js'), testOptions); testRunner.loadTests(require('./mouse.spec.js'), testOptions);
testRunner.loadTests(require('./network.spec.js'), testOptions); testRunner.loadTests(require('./navigation.spec.js'), testOptions);
testRunner.loadTests(require('./page.spec.js'), testOptions); testRunner.loadTests(require('./network.spec.js'), testOptions);
testRunner.loadTests(require('./queryselector.spec.js'), testOptions); testRunner.loadTests(require('./page.spec.js'), testOptions);
testRunner.loadTests(require('./screenshot.spec.js'), testOptions); testRunner.loadTests(require('./queryselector.spec.js'), testOptions);
testRunner.loadTests(require('./waittask.spec.js'), testOptions); testRunner.loadTests(require('./screenshot.spec.js'), testOptions);
testRunner.loadTests(require('./interception.spec.js'), testOptions); testRunner.loadTests(require('./waittask.spec.js'), testOptions);
testRunner.loadTests(require('./geolocation.spec.js'), testOptions); testRunner.loadTests(require('./interception.spec.js'), testOptions);
testRunner.loadTests(require('./workers.spec.js'), testOptions); testRunner.loadTests(require('./geolocation.spec.js'), testOptions);
testRunner.loadTests(require('./capabilities.spec.js'), testOptions); testRunner.loadTests(require('./workers.spec.js'), testOptions);
testRunner.loadTests(require('./capabilities.spec.js'), testOptions);
});
if (CHROMIUM) { describe.skip(!CHROMIUM)('[Chromium]', () => {
testRunner.loadTests(require('./chromium/chromium.spec.js'), testOptions); testRunner.loadTests(require('./chromium/chromium.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/coverage.spec.js'), testOptions); testRunner.loadTests(require('./chromium/coverage.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/pdf.spec.js'), testOptions); testRunner.loadTests(require('./chromium/pdf.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/session.spec.js'), testOptions); testRunner.loadTests(require('./chromium/session.spec.js'), testOptions);
} });
if (CHROMIUM || FFOX) { describe('[Permissions]', () => {
testRunner.loadTests(require('./features/permissions.spec.js'), testOptions); testRunner.loadTests(require('./features/permissions.spec.js'), testOptions);
} });
if (WEBKIT) {
testRunner.loadTests(require('./webkit/provisional.spec.js'), testOptions);
}
}); });
// Browser-level tests that are given a browser. // Browser-level tests that are given a browser.
testRunner.loadTests(require('./browser.spec.js'), testOptions); describe('[Driver]', () => {
testRunner.loadTests(require('./browsercontext.spec.js'), testOptions); testRunner.loadTests(require('./browser.spec.js'), testOptions);
testRunner.loadTests(require('./ignorehttpserrors.spec.js'), testOptions); testRunner.loadTests(require('./browsercontext.spec.js'), testOptions);
testRunner.loadTests(require('./popup.spec.js'), testOptions); testRunner.loadTests(require('./ignorehttpserrors.spec.js'), testOptions);
testRunner.loadTests(require('./popup.spec.js'), testOptions);
});
}); });
// Top-level tests that launch Browser themselves. // Top-level tests that launch Browser themselves.
testRunner.loadTests(require('./defaultbrowsercontext.spec.js'), testOptions); describe('[Driver]', () => {
testRunner.loadTests(require('./fixtures.spec.js'), testOptions); testRunner.loadTests(require('./defaultbrowsercontext.spec.js'), testOptions);
testRunner.loadTests(require('./launcher.spec.js'), testOptions); testRunner.loadTests(require('./fixtures.spec.js'), testOptions);
testRunner.loadTests(require('./headful.spec.js'), testOptions); testRunner.loadTests(require('./launcher.spec.js'), testOptions);
testRunner.loadTests(require('./multiclient.spec.js'), testOptions); testRunner.loadTests(require('./headful.spec.js'), testOptions);
testRunner.loadTests(require('./multiclient.spec.js'), testOptions);
});
if (CHROMIUM) { describe.skip(!CHROMIUM)('[Chromium]', () => {
testRunner.loadTests(require('./chromium/launcher.spec.js'), testOptions); testRunner.loadTests(require('./chromium/launcher.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/headful.spec.js'), testOptions); testRunner.loadTests(require('./chromium/headful.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/oopif.spec.js'), testOptions); testRunner.loadTests(require('./chromium/oopif.spec.js'), testOptions);
testRunner.loadTests(require('./chromium/tracing.spec.js'), testOptions); testRunner.loadTests(require('./chromium/tracing.spec.js'), testOptions);
} });
testRunner.loadTests(require('./web.spec.js'), testOptions); describe('[Driver]', () => {
testRunner.loadTests(require('./web.spec.js'), testOptions);
});
}; };

2
test/test.js
View file

@ -125,7 +125,7 @@ new Reporter(testRunner, {
verbose: process.argv.includes('--verbose'), verbose: process.argv.includes('--verbose'),
summary: !process.argv.includes('--verbose'), summary: !process.argv.includes('--verbose'),
showSlowTests: process.env.CI ? 5 : 0, showSlowTests: process.env.CI ? 5 : 0,
showSkippedTests: 10, showMarkedAsFailingTests: 10,
}); });
// await utils.initializeFlakinessDashboardIfNeeded(testRunner); // await utils.initializeFlakinessDashboardIfNeeded(testRunner);

2
test/web.spec.js
View file

@ -22,7 +22,7 @@ module.exports.describe = function({testRunner, expect, defaultBrowserOptions, p
const {it, fit, xit, dit} = testRunner; const {it, fit, xit, dit} = testRunner;
const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;
(CHROMIUM || FFOX) && describe('Web SDK', function() { describe('Web SDK', function() {
beforeAll(async state => { beforeAll(async state => {
state.controlledBrowserApp = await playwright.launchServer(defaultBrowserOptions); state.controlledBrowserApp = await playwright.launchServer(defaultBrowserOptions);
state.hostBrowser = await playwright.launch(defaultBrowserOptions); state.hostBrowser = await playwright.launch(defaultBrowserOptions);

32
utils/testrunner/Reporter.js
View file

@ -22,14 +22,14 @@ class Reporter {
constructor(runner, options = {}) { constructor(runner, options = {}) {
const { const {
showSlowTests = 3, showSlowTests = 3,
showSkippedTests = Infinity, showMarkedAsFailingTests = Infinity,
verbose = false, verbose = false,
summary = true, summary = true,
} = options; } = options;
this._filePathToLines = new Map(); this._filePathToLines = new Map();
this._runner = runner; this._runner = runner;
this._showSlowTests = showSlowTests; this._showSlowTests = showSlowTests;
this._showSkippedTests = showSkippedTests; this._showMarkedAsFailingTests = showMarkedAsFailingTests;
this._verbose = verbose; this._verbose = verbose;
this._summary = summary; this._summary = summary;
this._testCounter = 0; this._testCounter = 0;
@ -110,16 +110,17 @@ class Reporter {
} }
const skippedTests = this._runner.skippedTests(); const skippedTests = this._runner.skippedTests();
if (this._showSkippedTests && this._summary && skippedTests.length) { const markedAsFailingTests = this._runner.markedAsFailingTests();
if (skippedTests.length > 0) { if (this._showMarkedAsFailingTests && this._summary && markedAsFailingTests.length) {
console.log('\nSkipped:'); if (markedAsFailingTests.length > 0) {
skippedTests.slice(0, this._showSkippedTests).forEach((test, index) => { console.log('\nMarked as failing:');
markedAsFailingTests.slice(0, this._showMarkedAsFailingTests).forEach((test, index) => {
console.log(`${index + 1}) ${test.fullName} (${formatLocation(test.location)})`); console.log(`${index + 1}) ${test.fullName} (${formatLocation(test.location)})`);
}); });
} }
if (this._showSkippedTests < skippedTests.length) { if (this._showMarkedAsFailingTests < markedAsFailingTests.length) {
console.log(''); console.log('');
console.log(`... and ${colors.yellow(skippedTests.length - this._showSkippedTests)} more skipped tests ...`); console.log(`... and ${colors.yellow(markedAsFailingTests.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`);
} }
} }
@ -139,12 +140,14 @@ class Reporter {
const tests = this._runner.tests(); const tests = this._runner.tests();
const executedTests = tests.filter(test => test.result); const executedTests = tests.filter(test => test.result);
const okTestsLength = executedTests.length - failedTests.length - skippedTests.length; const okTestsLength = executedTests.length - failedTests.length - markedAsFailingTests.length - skippedTests.length;
let summaryText = ''; let summaryText = '';
if (failedTests.length || skippedTests.length) { if (failedTests.length || markedAsFailingTests.length) {
const summary = [`ok - ${colors.green(okTestsLength)}`]; const summary = [`ok - ${colors.green(okTestsLength)}`];
if (failedTests.length) if (failedTests.length)
summary.push(`failed - ${colors.red(failedTests.length)}`); summary.push(`failed - ${colors.red(failedTests.length)}`);
if (markedAsFailingTests.length)
summary.push(`marked as failing - ${colors.yellow(markedAsFailingTests.length)}`);
if (skippedTests.length) if (skippedTests.length)
summary.push(`skipped - ${colors.yellow(skippedTests.length)}`); summary.push(`skipped - ${colors.yellow(skippedTests.length)}`);
summaryText = ` (${summary.join(', ')})`; summaryText = ` (${summary.join(', ')})`;
@ -167,9 +170,11 @@ class Reporter {
this._printVerboseTestResult(this._testCounter, test, workerId); this._printVerboseTestResult(this._testCounter, test, workerId);
} else { } else {
if (test.result === 'ok') if (test.result === 'ok')
process.stdout.write(colors.green('.')); process.stdout.write(colors.green('\u00B7'));
else if (test.result === 'skipped') else if (test.result === 'skipped')
process.stdout.write(colors.yellow('*')); process.stdout.write(colors.yellow('\u00B7'));
else if (test.result === 'markedAsFailing')
process.stdout.write(colors.yellow('\u00D7'));
else if (test.result === 'failed') else if (test.result === 'failed')
process.stdout.write(colors.red('F')); process.stdout.write(colors.red('F'));
else if (test.result === 'crashed') else if (test.result === 'crashed')
@ -192,7 +197,8 @@ class Reporter {
} else if (test.result === 'crashed') { } else if (test.result === 'crashed') {
console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName} (${formatLocation(test.location)})`); console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'skipped') { } else if (test.result === 'skipped') {
console.log(`${prefix} ${colors.yellow('[SKIP]')} ${test.fullName} (${formatLocation(test.location)})`); } else if (test.result === 'markedAsFailing') {
console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'timedout') { } else if (test.result === 'timedout') {
console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout}ms]`)} ${test.fullName} (${formatLocation(test.location)})`); console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout}ms]`)} ${test.fullName} (${formatLocation(test.location)})`);
} else if (test.result === 'failed') { } else if (test.result === 'failed') {

42
utils/testrunner/TestRunner.js
View file

@ -72,13 +72,14 @@ const TestMode = {
Run: 'run', Run: 'run',
Skip: 'skip', Skip: 'skip',
Focus: 'focus', Focus: 'focus',
ExpectToFail: 'fail', MarkAsFailing: 'markAsFailing',
Flake: 'flake' Flake: 'flake'
}; };
const TestResult = { const TestResult = {
Ok: 'ok', Ok: 'ok',
ExpectToFail: 'skipped', // User marked as failed MarkedAsFailing: 'markedAsFailing', // User marked as failed
Skipped: 'skipped', // User marked as skipped
Failed: 'failed', // Exception happened during running Failed: 'failed', // Exception happened during running
TimedOut: 'timedout', // Timeout Exceeded while running TimedOut: 'timedout', // Timeout Exceeded while running
Terminated: 'terminated', // Execution terminated Terminated: 'terminated', // Execution terminated
@ -138,7 +139,7 @@ class TestPass {
for (let suite = test.suite; suite; suite = suite.parentSuite) for (let suite = test.suite; suite; suite = suite.parentSuite)
this._workerDistribution.set(suite, workerId); this._workerDistribution.set(suite, workerId);
// Do not shard skipped tests across workers. // Do not shard skipped tests across workers.
if (test.declaredMode !== TestMode.ExpectToFail) if (test.declaredMode !== TestMode.MarkAsFailing && test.declaredMode !== TestMode.Skip)
workerId = (workerId + 1) % parallel; workerId = (workerId + 1) % parallel;
} }
@ -200,8 +201,13 @@ class TestPass {
if (this._termination) if (this._termination)
return; return;
this._runner._willStartTest(test, workerId); this._runner._willStartTest(test, workerId);
if (test.declaredMode === TestMode.ExpectToFail) { if (test.declaredMode === TestMode.MarkAsFailing) {
test.result = TestResult.ExpectToFail; test.result = TestResult.MarkedAsFailing;
this._runner._didFinishTest(test, workerId);
return;
}
if (test.declaredMode === TestMode.Skip) {
test.result = TestResult.Skipped;
this._runner._didFinishTest(test, workerId); this._runner._didFinishTest(test, workerId);
return; return;
} }
@ -296,7 +302,7 @@ function specBuilder(action) {
}; };
func.fail = condition => { func.fail = condition => {
if (condition) if (condition)
mode = TestMode.ExpectToFail; mode = TestMode.MarkAsFailing;
return func; return func;
}; };
func.flake = condition => { func.flake = condition => {
@ -368,14 +374,20 @@ class TestRunner extends EventEmitter {
} }
_addTest(name, callback, mode, timeout) { _addTest(name, callback, mode, timeout) {
if (mode === TestMode.Skip)
return;
let suite = this._currentSuite; let suite = this._currentSuite;
let expectToFail = suite.declaredMode === TestMode.ExpectToFail; let markedAsFailing = suite.declaredMode === TestMode.MarkAsFailing;
while ((suite = suite.parentSuite)) while ((suite = suite.parentSuite))
expectToFail |= suite.declaredMode === TestMode.ExpectToFail; markedAsFailing |= suite.declaredMode === TestMode.MarkAsFailing;
if (expectToFail) if (markedAsFailing)
mode = TestMode.ExpectToFail; mode = TestMode.MarkAsFailing;
suite = this._currentSuite;
let skip = suite.declaredMode === TestMode.Skip;
while ((suite = suite.parentSuite))
skip |= suite.declaredMode === TestMode.Skip;
if (skip)
mode = TestMode.Skip;
const test = new Test(this._currentSuite, name, callback, mode, timeout); const test = new Test(this._currentSuite, name, callback, mode, timeout);
this._currentSuite.children.push(test); this._currentSuite.children.push(test);
this._tests.push(test); this._tests.push(test);
@ -384,8 +396,6 @@ class TestRunner extends EventEmitter {
} }
_addSuite(mode, name, callback, ...args) { _addSuite(mode, name, callback, ...args) {
if (mode === TestMode.Skip)
return;
const oldSuite = this._currentSuite; const oldSuite = this._currentSuite;
const suite = new Suite(this._currentSuite, name, mode); const suite = new Suite(this._currentSuite, name, mode);
this._currentSuite.children.push(suite); this._currentSuite.children.push(suite);
@ -491,6 +501,10 @@ class TestRunner extends EventEmitter {
return this._tests.filter(test => test.result === 'skipped'); return this._tests.filter(test => test.result === 'skipped');
} }
markedAsFailingTests() {
return this._tests.filter(test => test.result === 'markedAsFailing');
}
parallel() { parallel() {
return this._parallel; return this._parallel;
} }