devops: add headful linux bot (#2060)

.github/workflows/tests.yml

@@ -284,3 +284,24 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: bash test/installation-tests/installation-tests.sh
+
+  headful_linux:
+    name: "Headful Linux"
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 10
+    - uses: microsoft/playwright-github-action@v1
+    - run: npm ci
+    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
+    # Wrap `npm run` in a subshell to redirect STDERR to file.
+    - run: xvfb-run --auto-servernum -- bash -c "HEADLESS=false npm run test -- --line-break=100 2>./headful-linux-testrun.log"
+      env:
+        DEBUG: "*"
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: headful-linux-testrun.log
+        path: headful-linux-testrun.log

test/chromium/pdf.spec.js

@@ -19,7 +19,7 @@ const path = require('path');
 const {FFOX, CHROMIUM, WEBKIT, OUTPUT_DIR} = require('../utils').testOptions(browserType);
 
 // Printing to pdf is currently only supported in headless
-describe('Page.pdf', function() {
+describe.skip(!HEADLESS)('Page.pdf', function() {
   it('should be able to save file', async({page, server}) => {
     const outputFile = path.join(OUTPUT_DIR, 'output.pdf');
     await page.pdf({path: outputFile});

test/emulation.spec.js

@@ -555,7 +555,8 @@ describe('focus', function() {
     ]);
     expect(active).toEqual(['INPUT', 'TEXTAREA']);
   });
-  it('should not affect screenshots', async({page, server, golden}) => {
+  it.skip(FFOX && !HEADLESS)('should not affect screenshots', async({page, server, golden}) => {
+    // Firefox headful produces a different image.
     const page2 = await page.context().newPage();
     await Promise.all([
       page.setViewportSize({width: 500, height: 500}),

test/interception.spec.js

@@ -466,7 +466,8 @@ describe('Request.fulfill', function() {
     expect(response.statusText()).toBe('Unprocessable Entity');
     expect(await page.evaluate(() => document.body.textContent)).toBe('Yo, page!');
   });
-  it('should allow mocking binary responses', async({page, server, golden}) => {
+  it.skip(FFOX && !HEADLESS)('should allow mocking binary responses', async({page, server, golden}) => {
+    // Firefox headful produces a different image.
     await page.route('**/*', route => {
       const imageBuffer = fs.readFileSync(path.join(__dirname, 'assets', 'pptr.png'));
       route.fulfill({

test/queryselector.spec.js

@@ -19,6 +19,15 @@ const path = require('path');
 const zsSelectorEngineSource = require('../lib/generated/zsSelectorEngineSource');
 const {FFOX, CHROMIUM, WEBKIT} = require('./utils').testOptions(browserType);
 
+async function registerEngine(name, script, options) {
+  try {
+    await playwright.selectors.register(name, script, options);
+  } catch (e) {
+    if (!e.message.includes('has been already registered'))
+      throw e;
+  }
+}
+
 describe('Page.$eval', function() {
   it('should work with css selector', async({page, server}) => {
     await page.setContent('<section id="testAttribute">43543</section>');
@@ -437,12 +446,7 @@ describe('ElementHandle.$$ xpath', function() {
 
 describe('zselector', () => {
   beforeAll(async () => {
-    try {
-      await playwright.selectors.register('z', zsSelectorEngineSource.source);
-    } catch (e) {
-      if (!e.message.includes('has been already registered'))
-        throw e;
-    }
+    await registerEngine('z', zsSelectorEngineSource.source);
   });
 
   it('query', async ({page}) => {
@@ -744,7 +748,7 @@ describe('selectors.register', () => {
         return Array.from(root.querySelectorAll(selector));
       }
     });
-    await playwright.selectors.register('tag', `(${createTagSelector.toString()})()`);
+    await registerEngine('tag', `(${createTagSelector.toString()})()`);
     await page.setContent('<div><span></span></div><div></div>');
     expect(await playwright.selectors._createSelector('tag', await page.$('div'))).toBe('DIV');
     expect(await page.$eval('tag=DIV', e => e.nodeName)).toBe('DIV');
@@ -752,7 +756,7 @@ describe('selectors.register', () => {
     expect(await page.$$eval('tag=DIV', es => es.length)).toBe(2);
   });
   it('should work with path', async ({page}) => {
-    await playwright.selectors.register('foo', { path: path.join(__dirname, 'assets/sectionselectorengine.js') });
+    await registerEngine('foo', { path: path.join(__dirname, 'assets/sectionselectorengine.js') });
     await page.setContent('<section></section>');
     expect(await page.$eval('foo=whatever', e => e.nodeName)).toBe('SECTION');
   });
@@ -766,8 +770,8 @@ describe('selectors.register', () => {
         return [document.body, document.documentElement, window.__answer];
       }
     });
-    await playwright.selectors.register('main', createDummySelector);
-    await playwright.selectors.register('isolated', createDummySelector, { contentScript: true });
+    await registerEngine('main', createDummySelector);
+    await registerEngine('isolated', createDummySelector, { contentScript: true });
     await page.setContent('<div><span><section></section></span></div>');
     await page.evaluate(() => window.__answer = document.querySelector('span'));
     // Works in main if asked.
@@ -791,8 +795,8 @@ describe('selectors.register', () => {
     await page.setContent('<div><dummy id=d1></dummy></div><span><dummy id=d2></dummy></span>');
     expect(await page.$eval('div', e => e.nodeName)).toBe('DIV');
 
-    let error = await page.$('dummy=ignored').catch(e => e);
-    expect(error.message).toBe('Unknown engine "dummy" while parsing selector dummy=ignored');
+    let error = await page.$('neverregister=ignored').catch(e => e);
+    expect(error.message).toBe('Unknown engine "neverregister" while parsing selector neverregister=ignored');
 
     const createDummySelector = () => ({
       create(root, target) {
@@ -809,7 +813,7 @@ describe('selectors.register', () => {
     error = await playwright.selectors.register('$', createDummySelector).catch(e => e);
     expect(error.message).toBe('Selector engine name may only contain [a-zA-Z0-9_] characters');
 
-    await playwright.selectors.register('dummy', createDummySelector);
+    await registerEngine('dummy', createDummySelector);
     expect(await page.$eval('dummy=ignored', e => e.id)).toBe('d1');
     expect(await page.$eval('css=span >> dummy=ignored', e => e.id)).toBe('d2');
 

test/screenshot.spec.js

@@ -17,7 +17,10 @@
 
 const {FFOX, CHROMIUM, WEBKIT} = require('./utils').testOptions(browserType);
 
-describe('Page.screenshot', function() {
+// Firefox headful produces a different image.
+const ffheadful = FFOX && !HEADLESS;
+
+describe.skip(ffheadful)('Page.screenshot', function() {
   it('should work', async({page, server, golden}) => {
     await page.setViewportSize({width: 500, height: 500});
     await page.goto(server.PREFIX + '/grid.html');
@@ -221,7 +224,7 @@ describe('Page.screenshot', function() {
   });
 });
 
-describe('ElementHandle.screenshot', function() {
+describe.skip(ffheadful)('ElementHandle.screenshot', function() {
   it('should work', async({page, server, golden}) => {
     await page.setViewportSize({width: 500, height: 500});
     await page.goto(server.PREFIX + '/grid.html');

test/test.js

@@ -63,6 +63,7 @@ function collect(browserNames) {
     summary: !process.argv.includes('--verbose'),
     showSlowTests: process.env.CI ? 5 : 0,
     showMarkedAsFailingTests: 10,
+    lineBreak: parseInt(getCLIArgument('--line-break') || 0, 10),
   });
   if (config.setupTestRunner)
     config.setupTestRunner(testRunner);
@@ -171,6 +172,7 @@ function collect(browserNames) {
       // In addition to state, expose these two on global so that describes can access them.
       global.playwright = playwright;
       global.browserType = browserType;
+      global.HEADLESS = !!launchOptions.headless;
 
       testRunner.collector().useEnvironment(browserTypeEnvironment);
 
@@ -194,6 +196,7 @@ function collect(browserNames) {
         });
       }
 
+      delete global.HEADLESS;
       delete global.browserType;
       delete global.playwright;
     });

utils/testrunner/Reporter.js

@@ -26,6 +26,7 @@ class Reporter {
       showMarkedAsFailingTests = Infinity,
       verbose = false,
       summary = true,
+      lineBreak = 0,
     } = options;
     this._filePathToLines = new Map();
     this._delegate = delegate;
@@ -33,6 +34,7 @@ class Reporter {
     this._showMarkedAsFailingTests = showMarkedAsFailingTests;
     this._verbose = verbose;
     this._summary = summary;
+    this._lineBreak = lineBreak;
     this._testCounter = 0;
   }
 
@@ -157,8 +159,8 @@ class Reporter {
   }
 
   onTestRunFinished(testRun) {
+    ++this._testCounter;
     if (this._verbose) {
-      ++this._testCounter;
       this._printVerboseTestRunResult(this._testCounter, testRun);
     } else {
       if (testRun.result() === 'ok')
@@ -175,6 +177,8 @@ class Reporter {
         process.stdout.write(colors.magenta('.'));
       else if (testRun.result() === 'timedout')
         process.stdout.write(colors.red('T'));
+      if (this._lineBreak && !(this._testCounter % this._lineBreak))
+        process.stdout.write('\n');
     }
   }
 

utils/testrunner/index.js

@@ -39,6 +39,7 @@ class DefaultTestRunner {
       showMarkedAsFailingTests,
       verbose,
       summary,
+      lineBreak,
     } = options;
 
     this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI;
@@ -52,6 +53,7 @@ class DefaultTestRunner {
     this._showMarkedAsFailingTests = showMarkedAsFailingTests;
     this._verbose = verbose;
     this._summary = summary;
+    this._lineBreak = lineBreak;
 
     this._filter = new FocusedFilter();
     this._repeater = new Repeater();
@@ -126,6 +128,7 @@ class DefaultTestRunner {
         showMarkedAsFailingTests: this._showMarkedAsFailingTests,
         verbose: this._verbose,
         summary: this._summary,
+        lineBreak: this._lineBreak,
       };
       reporter = new Reporter(reporterDelegate, reporterOptions);
     }