diff --git a/test/playwright.spec.js b/test/playwright.spec.js index 5e25ea0420..95ef98009d 100644 --- a/test/playwright.spec.js +++ b/test/playwright.spec.js @@ -99,6 +99,16 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => { ASSETS_DIR, }; + function loadTests(modulePath) { + const module = require(modulePath); + if (typeof module.describe === 'function') + describe('', module.describe, testOptions); + if (typeof module.fdescribe === 'function') + fdescribe('', module.fdescribe, testOptions); + if (typeof module.xdescribe === 'function') + xdescribe('', module.xdescribe, testOptions); + } + describe('', function() { beforeAll(async state => { state.browser = await browserType.launch(defaultBrowserOptions); @@ -154,69 +164,69 @@ module.exports.describe = ({testRunner, product, playwrightPath}) => { // Page-level tests that are given a browser, a context and a page. // Each test is launched in a new browser context. - describe('[Accessibility]', () => testRunner.loadTests(require('./accessibility.spec.js'), testOptions)); + describe('[Accessibility]', () => loadTests('./accessibility.spec.js')); describe('[Driver]', () => { - testRunner.loadTests(require('./autowaiting.spec.js'), testOptions); - testRunner.loadTests(require('./click.spec.js'), testOptions); - testRunner.loadTests(require('./cookies.spec.js'), testOptions); - testRunner.loadTests(require('./dialog.spec.js'), testOptions); - testRunner.loadTests(require('./elementhandle.spec.js'), testOptions); - testRunner.loadTests(require('./emulation.spec.js'), testOptions); - testRunner.loadTests(require('./evaluation.spec.js'), testOptions); - testRunner.loadTests(require('./frame.spec.js'), testOptions); - testRunner.loadTests(require('./focus.spec.js'), testOptions); - testRunner.loadTests(require('./input.spec.js'), testOptions); - testRunner.loadTests(require('./jshandle.spec.js'), testOptions); - testRunner.loadTests(require('./keyboard.spec.js'), testOptions); - testRunner.loadTests(require('./mouse.spec.js'), testOptions); - testRunner.loadTests(require('./navigation.spec.js'), testOptions); - testRunner.loadTests(require('./network.spec.js'), testOptions); - testRunner.loadTests(require('./page.spec.js'), testOptions); - testRunner.loadTests(require('./queryselector.spec.js'), testOptions); - testRunner.loadTests(require('./screenshot.spec.js'), testOptions); - testRunner.loadTests(require('./waittask.spec.js'), testOptions); - testRunner.loadTests(require('./interception.spec.js'), testOptions); - testRunner.loadTests(require('./geolocation.spec.js'), testOptions); - testRunner.loadTests(require('./workers.spec.js'), testOptions); - testRunner.loadTests(require('./capabilities.spec.js'), testOptions); + loadTests('./autowaiting.spec.js'); + loadTests('./click.spec.js'); + loadTests('./cookies.spec.js'); + loadTests('./dialog.spec.js'); + loadTests('./elementhandle.spec.js'); + loadTests('./emulation.spec.js'); + loadTests('./evaluation.spec.js'); + loadTests('./frame.spec.js'); + loadTests('./focus.spec.js'); + loadTests('./input.spec.js'); + loadTests('./jshandle.spec.js'); + loadTests('./keyboard.spec.js'); + loadTests('./mouse.spec.js'); + loadTests('./navigation.spec.js'); + loadTests('./network.spec.js'); + loadTests('./page.spec.js'); + loadTests('./queryselector.spec.js'); + loadTests('./screenshot.spec.js'); + loadTests('./waittask.spec.js'); + loadTests('./interception.spec.js'); + loadTests('./geolocation.spec.js'); + loadTests('./workers.spec.js'); + loadTests('./capabilities.spec.js'); }); describe('[Permissions]', () => { - testRunner.loadTests(require('./permissions.spec.js'), testOptions); + loadTests('./permissions.spec.js'); }); describe.skip(!CHROMIUM)('[Chromium]', () => { - testRunner.loadTests(require('./chromium/chromium.spec.js'), testOptions); - testRunner.loadTests(require('./chromium/coverage.spec.js'), testOptions); - testRunner.loadTests(require('./chromium/pdf.spec.js'), testOptions); - testRunner.loadTests(require('./chromium/session.spec.js'), testOptions); + loadTests('./chromium/chromium.spec.js'); + loadTests('./chromium/coverage.spec.js'); + loadTests('./chromium/pdf.spec.js'); + loadTests('./chromium/session.spec.js'); }); }); // Browser-level tests that are given a browser. describe('[Driver]', () => { - testRunner.loadTests(require('./browser.spec.js'), testOptions); - testRunner.loadTests(require('./browsercontext.spec.js'), testOptions); - testRunner.loadTests(require('./ignorehttpserrors.spec.js'), testOptions); - testRunner.loadTests(require('./popup.spec.js'), testOptions); + loadTests('./browser.spec.js'); + loadTests('./browsercontext.spec.js'); + loadTests('./ignorehttpserrors.spec.js'); + loadTests('./popup.spec.js'); }); }); // Top-level tests that launch Browser themselves. describe('[Driver]', () => { - testRunner.loadTests(require('./defaultbrowsercontext.spec.js'), testOptions); - testRunner.loadTests(require('./fixtures.spec.js'), testOptions); - testRunner.loadTests(require('./launcher.spec.js'), testOptions); - testRunner.loadTests(require('./headful.spec.js'), testOptions); - testRunner.loadTests(require('./multiclient.spec.js'), testOptions); + loadTests('./defaultbrowsercontext.spec.js'); + loadTests('./fixtures.spec.js'); + loadTests('./launcher.spec.js'); + loadTests('./headful.spec.js'); + loadTests('./multiclient.spec.js'); }); describe.skip(!CHROMIUM)('[Chromium]', () => { - testRunner.loadTests(require('./chromium/launcher.spec.js'), testOptions); - testRunner.loadTests(require('./chromium/oopif.spec.js'), testOptions); - testRunner.loadTests(require('./chromium/tracing.spec.js'), testOptions); + loadTests('./chromium/launcher.spec.js'); + loadTests('./chromium/oopif.spec.js'); + loadTests('./chromium/tracing.spec.js'); }); describe('[Driver]', () => { - testRunner.loadTests(require('./web.spec.js'), testOptions); + loadTests('./web.spec.js'); }); }; diff --git a/test/test.js b/test/test.js index 75589dabeb..7f498da98e 100644 --- a/test/test.js +++ b/test/test.js @@ -18,6 +18,7 @@ const path = require('path'); const {TestServer} = require('../utils/testserver/'); const {TestRunner, Reporter} = require('../utils/testrunner/'); const utils = require('./utils'); +const inspector = require('inspector'); let parallel = 1; if (process.env.PW_PARALLEL_TESTS) @@ -30,11 +31,66 @@ require('events').defaultMaxListeners *= parallel; let timeout = process.env.CI ? 30 * 1000 : 10 * 1000; if (!isNaN(process.env.TIMEOUT)) timeout = parseInt(process.env.TIMEOUT * 1000, 10); +const MAJOR_NODEJS_VERSION = parseInt(process.version.substring(1).split('.')[0], 10); +if (MAJOR_NODEJS_VERSION >= 8 && inspector.url()) { + console.log('Detected inspector - disabling timeout to be debugger-friendly'); + timeout = 0; +} + const testRunner = new TestRunner({ timeout, parallel, breakOnFailure: process.argv.indexOf('--break-on-failure') !== -1, + installCommonHelpers: false }); +testRunner.testModifier('skip', (t, condition) => condition && t.setMode(t.Modes.Skip)); +testRunner.suiteModifier('skip', (s, condition) => condition && s.setMode(s.Modes.Skip)); +testRunner.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); +testRunner.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); +testRunner.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3)); +testRunner.testModifier('repeat', (t, count) => t.setRepeat(count)); +testRunner.suiteModifier('repeat', (s, count) => s.setRepeat(count)); +testRunner.testAttribute('focus', t => t.setMode(t.Modes.Focus)); +testRunner.suiteAttribute('focus', s => s.setMode(s.Modes.Focus)); +testRunner.testAttribute('debug', t => { + t.setMode(t.Modes.Focus); + t.setTimeout(100000000); + + let session; + t.before(async () => { + const util = require('util'); + const fs = require('fs'); + const url = require('url'); + const readFileAsync = util.promisify(fs.readFile.bind(fs)); + session = new inspector.Session(); + session.connect(); + const postAsync = util.promisify(session.post.bind(session)); + await postAsync('Debugger.enable'); + const setBreakpointCommands = []; + const N = t.body().toString().split('\n').length; + const location = t.location(); + const lines = (await readFileAsync(location.filePath, 'utf8')).split('\n'); + for (let line = 0; line < N; ++line) { + const lineNumber = line + location.lineNumber; + setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', { + url: url.pathToFileURL(location.filePath), + lineNumber, + condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`, + }).catch(e => {})); + } + await Promise.all(setBreakpointCommands); + }); + + t.after(async () => { + session.disconnect(); + }); +}); +testRunner.fdescribe = testRunner.describe.focus; +testRunner.xdescribe = testRunner.describe.skip(true); +testRunner.fit = testRunner.it.focus; +testRunner.xit = testRunner.it.skip(true); +testRunner.dit = testRunner.it.debug; + const {describe, fdescribe, beforeAll, afterAll, beforeEach, afterEach} = testRunner; console.log('Testing on Node', process.version); @@ -106,7 +162,7 @@ for (const browserConfig of BROWSER_CONFIGS) { continue; const product = browserConfig.name; describe(product, () => { - testRunner.loadTests(require('./playwright.spec.js'), { + testRunner.describe('', require('./playwright.spec.js').describe, { product, playwrightPath: utils.projectRoot(), testRunner, diff --git a/test/types.d.ts b/test/types.d.ts index 65aedd5586..10f20ead45 100644 --- a/test/types.d.ts +++ b/test/types.d.ts @@ -36,7 +36,7 @@ type TestRunner = { fit: ItFunction; dit: ItFunction; - beforeAll, beforeEach, afterAll, afterEach, loadTests; + beforeAll, beforeEach, afterAll, afterEach; }; interface TestSetup { diff --git a/utils/testrunner/Multimap.js b/utils/testrunner/Multimap.js deleted file mode 100644 index 6e9c0769a6..0000000000 --- a/utils/testrunner/Multimap.js +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -class Multimap { - constructor() { - this._map = new Map(); - } - - set(key, value) { - let set = this._map.get(key); - if (!set) { - set = new Set(); - this._map.set(key, set); - } - set.add(value); - } - - get(key) { - let result = this._map.get(key); - if (!result) - result = new Set(); - return result; - } - - has(key) { - return this._map.has(key); - } - - hasValue(key, value) { - const set = this._map.get(key); - if (!set) - return false; - return set.has(value); - } - - /** - * @return {number} - */ - get size() { - return this._map.size; - } - - delete(key, value) { - const values = this.get(key); - const result = values.delete(value); - if (!values.size) - this._map.delete(key); - return result; - } - - deleteAll(key) { - this._map.delete(key); - } - - firstValue(key) { - const set = this._map.get(key); - if (!set) - return null; - return set.values().next().value; - } - - firstKey() { - return this._map.keys().next().value; - } - - valuesArray() { - const result = []; - for (const key of this._map.keys()) - result.push(...Array.from(this._map.get(key).values())); - return result; - } - - keysArray() { - return Array.from(this._map.keys()); - } - - clear() { - this._map.clear(); - } -} - -module.exports = Multimap; diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index 34faf68b71..bbdf6bbd15 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -14,25 +14,15 @@ * limitations under the License. */ -const util = require('util'); -const url = require('url'); -const inspector = require('inspector'); const EventEmitter = require('events'); -const Multimap = require('./Multimap'); -const fs = require('fs'); const {SourceMapSupport} = require('./SourceMapSupport'); const debug = require('debug'); const {getCallerLocation} = require('./utils'); const INFINITE_TIMEOUT = 100000000; - -const readFileAsync = util.promisify(fs.readFile.bind(fs)); - const TimeoutError = new Error('Timeout'); const TerminatedError = new Error('Terminated'); -const MAJOR_NODEJS_VERSION = parseInt(process.version.substring(1).split('.')[0], 10); - function runUserCallback(callback, timeout, args) { let terminateCallback; let timeoutId; @@ -609,7 +599,7 @@ class TestRunner extends EventEmitter { parallel = 1, breakOnFailure = false, crashIfTestsAreFocusedOnCI = true, - disableTimeoutWhenInspectorIsEnabled = true, + installCommonHelpers = true, } = options; this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI; this._sourceMapSupport = new SourceMapSupport(); @@ -626,16 +616,6 @@ class TestRunner extends EventEmitter { this._testModifiers = new Map(); this._testAttributes = new Map(); - if (MAJOR_NODEJS_VERSION >= 8 && disableTimeoutWhenInspectorIsEnabled) { - if (inspector.url()) { - console.log('TestRunner detected inspector; overriding certain properties to be debugger-friendly'); - console.log(' - timeout = 0 (Infinite)'); - this._timeout = INFINITE_TIMEOUT; - } - } - - this._debuggerLogBreakpointLines = new Multimap(); - this.beforeAll = (callback) => this._currentSuite.beforeAll(callback); this.beforeEach = (callback) => this._currentSuite.beforeEach(callback); this.afterAll = (callback) => this._currentSuite.afterAll(callback); @@ -644,29 +624,12 @@ class TestRunner extends EventEmitter { this.describe = this._suiteBuilder([]); this.it = this._testBuilder([]); - this.testAttribute('debug', t => { - t.setMode(t.Modes.Focus); - t.setTimeout(INFINITE_TIMEOUT); - const N = t.body().toString().split('\n').length; - const location = t.location(); - for (let line = 0; line < N; ++line) - this._debuggerLogBreakpointLines.set(location.filePath, line + location.lineNumber); - }); - - this.testModifier('skip', (t, condition) => condition && t.setMode(t.Modes.Skip)); - this.suiteModifier('skip', (s, condition) => condition && s.setMode(s.Modes.Skip)); - this.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); - this.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); - this.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3)); - this.testModifier('repeat', (t, count) => t.setRepeat(count)); - this.suiteModifier('repeat', (s, count) => s.setRepeat(count)); - this.testAttribute('focus', t => t.setMode(t.Modes.Focus)); - this.suiteAttribute('focus', s => s.setMode(s.Modes.Focus)); - this.fdescribe = this.describe.focus; - this.xdescribe = this.describe.skip(true); - this.fit = this.it.focus; - this.xit = this.it.skip(true); - this.dit = this.it.debug; + if (installCommonHelpers) { + this.fdescribe = this.describe.setup(s => s.setMode(s.Modes.Focus)); + this.xdescribe = this.describe.setup(s => s.setMode(s.Modes.Skip)); + this.fit = this.it.setup(t => t.setMode(t.Modes.Focus)); + this.xit = this.it.setup(t => t.setMode(t.Modes.Skip)); + } } _suiteBuilder(callbacks) { @@ -683,6 +646,8 @@ class TestRunner extends EventEmitter { } }, { get: (obj, prop) => { + if (prop === 'setup') + return callback => this._suiteBuilder([...callbacks, { callback, args: [] }]); if (this._suiteModifiers.has(prop)) return (...args) => this._suiteBuilder([...callbacks, { callback: this._suiteModifiers.get(prop), args }]); if (this._suiteAttributes.has(prop)) @@ -703,6 +668,8 @@ class TestRunner extends EventEmitter { this._tests.push(test._clone()); }, { get: (obj, prop) => { + if (prop === 'setup') + return callback => this._testBuilder([...callbacks, { callback, args: [] }]); if (this._testModifiers.has(prop)) return (...args) => this._testBuilder([...callbacks, { callback: this._testModifiers.get(prop), args }]); if (this._testAttributes.has(prop)) @@ -728,18 +695,8 @@ class TestRunner extends EventEmitter { this._suiteAttributes.set(name, callback); } - loadTests(module, ...args) { - if (typeof module.describe === 'function') - this.describe('', module.describe, ...args); - if (typeof module.fdescribe === 'function') - this.describe.focus('', module.fdescribe, ...args); - if (typeof module.xdescribe === 'function') - this.describe.skip(true)('', module.xdescribe, ...args); - } - async run(options = {}) { const { totalTimeout = 0 } = options; - let session = this._debuggerLogBreakpointLines.size ? await setLogBreakpoints(this._debuggerLogBreakpointLines) : null; const runnableTests = this.runnableTests(); this.emit(TestRunner.Events.Started, runnableTests); @@ -762,8 +719,6 @@ class TestRunner extends EventEmitter { } } this.emit(TestRunner.Events.Finished, result); - if (session) - session.disconnect(); return result; } @@ -844,36 +799,6 @@ class TestRunner extends EventEmitter { } } -async function setLogBreakpoints(debuggerLogBreakpoints) { - const session = new inspector.Session(); - session.connect(); - const postAsync = util.promisify(session.post.bind(session)); - await postAsync('Debugger.enable'); - const setBreakpointCommands = []; - for (const filePath of debuggerLogBreakpoints.keysArray()) { - const lineNumbers = debuggerLogBreakpoints.get(filePath); - const lines = (await readFileAsync(filePath, 'utf8')).split('\n'); - for (const lineNumber of lineNumbers) { - setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', { - url: url.pathToFileURL(filePath), - lineNumber, - condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`, - }).catch(e => {})); - }; - } - await Promise.all(setBreakpointCommands); - return session; -} - -/** - * @param {*} value - * @param {string=} message - */ -function assert(value, message) { - if (!value) - throw new Error(message); -} - TestRunner.Events = { Started: 'started', Finished: 'finished', diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js index f671bf7094..a76644dfbe 100644 --- a/utils/testrunner/test/testrunner.spec.js +++ b/utils/testrunner/test/testrunner.spec.js @@ -54,7 +54,9 @@ module.exports.addTests = function({testRunner, expect}) { it('should run a failed focused test', async() => { const t = newTestRunner(); let run = false; - t.fit.fail(true)('uno', () => { run = true; throw new Error('failure'); }); + t.fit.setup(t => t.setExpectation(t.Expectations.Fail))('uno', () => { + run = true; throw new Error('failure'); + }); expect(t.tests().length).toBe(1); await t.run(); expect(run).toBe(true); @@ -374,6 +376,8 @@ module.exports.addTests = function({testRunner, expect}) { }); it('should handle repeat', async() => { const t = newTestRunner(); + t.testModifier('repeat', (t, count) => t.setRepeat(count)); + t.suiteModifier('repeat', (s, count) => s.setRepeat(count)); let suite = 0; let test = 0; let beforeAll = 0;