From 963dc72dd222258d9708186a803bdd3e53ee765d Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Mon, 4 May 2020 15:15:51 -0700 Subject: [PATCH] devops: add headful linux bot (#2060) --- .github/workflows/tests.yml | 21 +++++++++++++++++++++ test/chromium/pdf.spec.js | 2 +- test/emulation.spec.js | 3 ++- test/interception.spec.js | 3 ++- test/queryselector.spec.js | 30 +++++++++++++++++------------- test/screenshot.spec.js | 7 +++++-- test/test.js | 3 +++ utils/testrunner/Reporter.js | 6 +++++- utils/testrunner/index.js | 3 +++ 9 files changed, 59 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d070e90959..ded9dcfa81 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -284,3 +284,24 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: bash test/installation-tests/installation-tests.sh + + headful_linux: + name: "Headful Linux" + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: 10 + - uses: microsoft/playwright-github-action@v1 + - run: npm ci + # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR + # Wrap `npm run` in a subshell to redirect STDERR to file. + - run: xvfb-run --auto-servernum -- bash -c "HEADLESS=false npm run test -- --line-break=100 2>./headful-linux-testrun.log" + env: + DEBUG: "*" + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: headful-linux-testrun.log + path: headful-linux-testrun.log diff --git a/test/chromium/pdf.spec.js b/test/chromium/pdf.spec.js index 3a5cf33614..d812aeeb14 100644 --- a/test/chromium/pdf.spec.js +++ b/test/chromium/pdf.spec.js @@ -19,7 +19,7 @@ const path = require('path'); const {FFOX, CHROMIUM, WEBKIT, OUTPUT_DIR} = require('../utils').testOptions(browserType); // Printing to pdf is currently only supported in headless -describe('Page.pdf', function() { +describe.skip(!HEADLESS)('Page.pdf', function() { it('should be able to save file', async({page, server}) => { const outputFile = path.join(OUTPUT_DIR, 'output.pdf'); await page.pdf({path: outputFile}); diff --git a/test/emulation.spec.js b/test/emulation.spec.js index c0f7feaf6a..1b95369f99 100644 --- a/test/emulation.spec.js +++ b/test/emulation.spec.js @@ -555,7 +555,8 @@ describe('focus', function() { ]); expect(active).toEqual(['INPUT', 'TEXTAREA']); }); - it('should not affect screenshots', async({page, server, golden}) => { + it.skip(FFOX && !HEADLESS)('should not affect screenshots', async({page, server, golden}) => { + // Firefox headful produces a different image. const page2 = await page.context().newPage(); await Promise.all([ page.setViewportSize({width: 500, height: 500}), diff --git a/test/interception.spec.js b/test/interception.spec.js index d44bd3a71f..fc6173ea49 100644 --- a/test/interception.spec.js +++ b/test/interception.spec.js @@ -466,7 +466,8 @@ describe('Request.fulfill', function() { expect(response.statusText()).toBe('Unprocessable Entity'); expect(await page.evaluate(() => document.body.textContent)).toBe('Yo, page!'); }); - it('should allow mocking binary responses', async({page, server, golden}) => { + it.skip(FFOX && !HEADLESS)('should allow mocking binary responses', async({page, server, golden}) => { + // Firefox headful produces a different image. await page.route('**/*', route => { const imageBuffer = fs.readFileSync(path.join(__dirname, 'assets', 'pptr.png')); route.fulfill({ diff --git a/test/queryselector.spec.js b/test/queryselector.spec.js index 82977cea9a..1acc6014f9 100644 --- a/test/queryselector.spec.js +++ b/test/queryselector.spec.js @@ -19,6 +19,15 @@ const path = require('path'); const zsSelectorEngineSource = require('../lib/generated/zsSelectorEngineSource'); const {FFOX, CHROMIUM, WEBKIT} = require('./utils').testOptions(browserType); +async function registerEngine(name, script, options) { + try { + await playwright.selectors.register(name, script, options); + } catch (e) { + if (!e.message.includes('has been already registered')) + throw e; + } +} + describe('Page.$eval', function() { it('should work with css selector', async({page, server}) => { await page.setContent('
43543
'); @@ -437,12 +446,7 @@ describe('ElementHandle.$$ xpath', function() { describe('zselector', () => { beforeAll(async () => { - try { - await playwright.selectors.register('z', zsSelectorEngineSource.source); - } catch (e) { - if (!e.message.includes('has been already registered')) - throw e; - } + await registerEngine('z', zsSelectorEngineSource.source); }); it('query', async ({page}) => { @@ -744,7 +748,7 @@ describe('selectors.register', () => { return Array.from(root.querySelectorAll(selector)); } }); - await playwright.selectors.register('tag', `(${createTagSelector.toString()})()`); + await registerEngine('tag', `(${createTagSelector.toString()})()`); await page.setContent('
'); expect(await playwright.selectors._createSelector('tag', await page.$('div'))).toBe('DIV'); expect(await page.$eval('tag=DIV', e => e.nodeName)).toBe('DIV'); @@ -752,7 +756,7 @@ describe('selectors.register', () => { expect(await page.$$eval('tag=DIV', es => es.length)).toBe(2); }); it('should work with path', async ({page}) => { - await playwright.selectors.register('foo', { path: path.join(__dirname, 'assets/sectionselectorengine.js') }); + await registerEngine('foo', { path: path.join(__dirname, 'assets/sectionselectorengine.js') }); await page.setContent('
'); expect(await page.$eval('foo=whatever', e => e.nodeName)).toBe('SECTION'); }); @@ -766,8 +770,8 @@ describe('selectors.register', () => { return [document.body, document.documentElement, window.__answer]; } }); - await playwright.selectors.register('main', createDummySelector); - await playwright.selectors.register('isolated', createDummySelector, { contentScript: true }); + await registerEngine('main', createDummySelector); + await registerEngine('isolated', createDummySelector, { contentScript: true }); await page.setContent('
'); await page.evaluate(() => window.__answer = document.querySelector('span')); // Works in main if asked. @@ -791,8 +795,8 @@ describe('selectors.register', () => { await page.setContent('
'); expect(await page.$eval('div', e => e.nodeName)).toBe('DIV'); - let error = await page.$('dummy=ignored').catch(e => e); - expect(error.message).toBe('Unknown engine "dummy" while parsing selector dummy=ignored'); + let error = await page.$('neverregister=ignored').catch(e => e); + expect(error.message).toBe('Unknown engine "neverregister" while parsing selector neverregister=ignored'); const createDummySelector = () => ({ create(root, target) { @@ -809,7 +813,7 @@ describe('selectors.register', () => { error = await playwright.selectors.register('$', createDummySelector).catch(e => e); expect(error.message).toBe('Selector engine name may only contain [a-zA-Z0-9_] characters'); - await playwright.selectors.register('dummy', createDummySelector); + await registerEngine('dummy', createDummySelector); expect(await page.$eval('dummy=ignored', e => e.id)).toBe('d1'); expect(await page.$eval('css=span >> dummy=ignored', e => e.id)).toBe('d2'); diff --git a/test/screenshot.spec.js b/test/screenshot.spec.js index 497636f8d7..36801caf58 100644 --- a/test/screenshot.spec.js +++ b/test/screenshot.spec.js @@ -17,7 +17,10 @@ const {FFOX, CHROMIUM, WEBKIT} = require('./utils').testOptions(browserType); -describe('Page.screenshot', function() { +// Firefox headful produces a different image. +const ffheadful = FFOX && !HEADLESS; + +describe.skip(ffheadful)('Page.screenshot', function() { it('should work', async({page, server, golden}) => { await page.setViewportSize({width: 500, height: 500}); await page.goto(server.PREFIX + '/grid.html'); @@ -221,7 +224,7 @@ describe('Page.screenshot', function() { }); }); -describe('ElementHandle.screenshot', function() { +describe.skip(ffheadful)('ElementHandle.screenshot', function() { it('should work', async({page, server, golden}) => { await page.setViewportSize({width: 500, height: 500}); await page.goto(server.PREFIX + '/grid.html'); diff --git a/test/test.js b/test/test.js index 16f4d11318..526393d9aa 100644 --- a/test/test.js +++ b/test/test.js @@ -63,6 +63,7 @@ function collect(browserNames) { summary: !process.argv.includes('--verbose'), showSlowTests: process.env.CI ? 5 : 0, showMarkedAsFailingTests: 10, + lineBreak: parseInt(getCLIArgument('--line-break') || 0, 10), }); if (config.setupTestRunner) config.setupTestRunner(testRunner); @@ -171,6 +172,7 @@ function collect(browserNames) { // In addition to state, expose these two on global so that describes can access them. global.playwright = playwright; global.browserType = browserType; + global.HEADLESS = !!launchOptions.headless; testRunner.collector().useEnvironment(browserTypeEnvironment); @@ -194,6 +196,7 @@ function collect(browserNames) { }); } + delete global.HEADLESS; delete global.browserType; delete global.playwright; }); diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js index 69e031fd41..ff88b4ca56 100644 --- a/utils/testrunner/Reporter.js +++ b/utils/testrunner/Reporter.js @@ -26,6 +26,7 @@ class Reporter { showMarkedAsFailingTests = Infinity, verbose = false, summary = true, + lineBreak = 0, } = options; this._filePathToLines = new Map(); this._delegate = delegate; @@ -33,6 +34,7 @@ class Reporter { this._showMarkedAsFailingTests = showMarkedAsFailingTests; this._verbose = verbose; this._summary = summary; + this._lineBreak = lineBreak; this._testCounter = 0; } @@ -157,8 +159,8 @@ class Reporter { } onTestRunFinished(testRun) { + ++this._testCounter; if (this._verbose) { - ++this._testCounter; this._printVerboseTestRunResult(this._testCounter, testRun); } else { if (testRun.result() === 'ok') @@ -175,6 +177,8 @@ class Reporter { process.stdout.write(colors.magenta('.')); else if (testRun.result() === 'timedout') process.stdout.write(colors.red('T')); + if (this._lineBreak && !(this._testCounter % this._lineBreak)) + process.stdout.write('\n'); } } diff --git a/utils/testrunner/index.js b/utils/testrunner/index.js index 5f053fd9ab..ca27554fee 100644 --- a/utils/testrunner/index.js +++ b/utils/testrunner/index.js @@ -39,6 +39,7 @@ class DefaultTestRunner { showMarkedAsFailingTests, verbose, summary, + lineBreak, } = options; this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI; @@ -52,6 +53,7 @@ class DefaultTestRunner { this._showMarkedAsFailingTests = showMarkedAsFailingTests; this._verbose = verbose; this._summary = summary; + this._lineBreak = lineBreak; this._filter = new FocusedFilter(); this._repeater = new Repeater(); @@ -126,6 +128,7 @@ class DefaultTestRunner { showMarkedAsFailingTests: this._showMarkedAsFailingTests, verbose: this._verbose, summary: this._summary, + lineBreak: this._lineBreak, }; reporter = new Reporter(reporterDelegate, reporterOptions); }