From 9116adc6841c132292b6abbfbf770f1fd61574a5 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 7 Feb 2022 10:41:56 -0800 Subject: [PATCH] chore: allow opt-into the legacy global setup mode (#11888) --- packages/playwright-test/src/runner.ts | 97 ++-- tests/playwright-test/global-setup.spec.ts | 497 +++++++++++---------- 2 files changed, 316 insertions(+), 278 deletions(-) diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 60fe9ad9b5..8d7a6ef99a 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -140,6 +140,14 @@ export class Runner { async runAllTests(options: RunOptions = {}): Promise { this._reporter = await this._createReporter(!!options.listOnly); const config = this._loader.fullConfig(); + + let legacyGlobalTearDown: (() => Promise) | undefined; + if (process.env.PW_TEST_LEGACY_GLOBAL_SETUP_MODE) { + legacyGlobalTearDown = await this._performGlobalSetup(config); + if (!legacyGlobalTearDown) + return { status: 'failed' }; + } + const result = await raceAgainstTimeout(() => this._run(!!options.listOnly, options.filePatternFilter || [], options.projectFilter), config.globalTimeout); let fullResult: FullResult; if (result.timedOut) { @@ -149,6 +157,7 @@ export class Runner { fullResult = result.result; } await this._reporter.onEnd?.(fullResult); + await legacyGlobalTearDown?.(); // Calling process.exit() might truncate large stdout/stderr output. // See https://github.com/nodejs/node/issues/6456. @@ -351,21 +360,19 @@ export class Runner { if (list) return { status: 'passed' }; - // 13. Declare global setup to tear down in finally. - const internalGlobalTeardowns: (() => Promise)[] = []; - let webServer: WebServer | undefined; - let globalSetupResult: any; + + // 13. Run Global setup. + let globalTearDown: (() => Promise) | undefined; + if (!process.env.PW_TEST_LEGACY_GLOBAL_SETUP_MODE) { + globalTearDown = await this._performGlobalSetup(config); + if (!globalTearDown) + return { status: 'failed' }; + } const result: FullResult = { status: 'passed' }; + // 14. Run tests. try { - // 14. Perform global setup. - for (const internalGlobalSetup of this._internalGlobalSetups) - internalGlobalTeardowns.push(await internalGlobalSetup()); - webServer = config.webServer ? await WebServer.create(config.webServer) : undefined; - if (config.globalSetup) - globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig()); - const sigintWatcher = new SigIntWatcher(); let hasWorkerErrors = false; @@ -389,35 +396,59 @@ export class Runner { this._reporter.onError?.(serializeError(e)); return { status: 'failed' }; } finally { - - await this._runAndAssignError(async () => { - if (globalSetupResult && typeof globalSetupResult === 'function') - await globalSetupResult(this._loader.fullConfig()); - }, result); - - await this._runAndAssignError(async () => { - if (config.globalTeardown) - await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig()); - }, result); - - await this._runAndAssignError(async () => { - await webServer?.kill(); - }, result); - - await this._runAndAssignError(async () => { - for (const internalGlobalTeardown of internalGlobalTeardowns) - await internalGlobalTeardown(); - }, result); + await globalTearDown?.(); } return result; } - private async _runAndAssignError(callback: () => Promise, result: FullResult) { + private async _performGlobalSetup(config: FullConfig): Promise<(() => Promise) | undefined> { + const result: FullResult = { status: 'passed' }; + const internalGlobalTeardowns: (() => Promise)[] = []; + let globalSetupResult: any; + let webServer: WebServer | undefined; + + const tearDown = async () => { + await this._runAndReportError(async () => { + if (globalSetupResult && typeof globalSetupResult === 'function') + await globalSetupResult(this._loader.fullConfig()); + }, result); + + await this._runAndReportError(async () => { + if (config.globalTeardown) + await (await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown'))(this._loader.fullConfig()); + }, result); + + await this._runAndReportError(async () => { + await webServer?.kill(); + }, result); + + await this._runAndReportError(async () => { + for (const internalGlobalTeardown of internalGlobalTeardowns) + await internalGlobalTeardown(); + }, result); + }; + + await this._runAndReportError(async () => { + for (const internalGlobalSetup of this._internalGlobalSetups) + internalGlobalTeardowns.push(await internalGlobalSetup()); + webServer = config.webServer ? await WebServer.create(config.webServer) : undefined; + if (config.globalSetup) + globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig()); + }, result); + + if (result.status !== 'passed') { + tearDown(); + return; + } + + return tearDown; + } + + private async _runAndReportError(callback: () => Promise, result: FullResult) { try { await callback(); } catch (e) { - if (result.status === 'passed') - result.status = 'failed'; + result.status = 'failed'; this._reporter.onError?.(serializeError(e)); } } diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 24822c9dd1..c82fbc7f07 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -16,252 +16,259 @@ import { test, expect } from './playwright-test-fixtures'; -test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => { - const { results, output } = await runInlineTest({ - 'playwright.config.ts': ` - import * as path from 'path'; - module.exports = { - globalSetup: './globalSetup', - globalTeardown: path.join(__dirname, 'globalTeardown.ts'), - }; - `, - 'globalSetup.ts': ` - module.exports = async () => { - await new Promise(f => setTimeout(f, 100)); - global.value = 42; - process.env.FOO = String(global.value); - }; - `, - 'globalTeardown.ts': ` - module.exports = async () => { - console.log('teardown=' + global.value); - }; - `, - 'a.test.js': ` - const { test } = pwt; - test('should work', async ({}, testInfo) => { - expect(process.env.FOO).toBe('42'); - }); - `, - }); - expect(results[0].status).toBe('passed'); - expect(output).toContain('teardown=42'); -}); +for (const mode of ['legacy', 'default']) { + test.describe(`${mode} mode`, () => { + const env = { PW_TEST_LEGACY_GLOBAL_SETUP_MODE: mode === 'legacy' ? '1' : undefined }; -test('globalTeardown runs after failures', async ({ runInlineTest }) => { - const { results, output } = await runInlineTest({ - 'playwright.config.ts': ` - import * as path from 'path'; - module.exports = { - globalSetup: 'globalSetup.ts', - globalTeardown: './globalTeardown.ts', - }; - `, - 'globalSetup.ts': ` - module.exports = async () => { - await new Promise(f => setTimeout(f, 100)); - global.value = 42; - process.env.FOO = String(global.value); - }; - `, - 'globalTeardown.ts': ` - module.exports = async () => { - console.log('teardown=' + global.value); - }; - `, - 'a.test.js': ` - const { test } = pwt; - test('should work', async ({}, testInfo) => { - expect(process.env.FOO).toBe('43'); - }); - `, - }); - expect(results[0].status).toBe('failed'); - expect(output).toContain('teardown=42'); -}); - -test('globalTeardown does not run when globalSetup times out', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - import * as path from 'path'; - module.exports = { - globalSetup: './globalSetup.ts', - globalTeardown: 'globalTeardown.ts', - globalTimeout: 1000, - }; - `, - 'globalSetup.ts': ` - module.exports = async () => { - await new Promise(f => setTimeout(f, 10000)); - }; - `, - 'globalTeardown.ts': ` - module.exports = async () => { - console.log('teardown='); - }; - `, - 'a.test.js': ` - const { test } = pwt; - test('should not run', async ({}, testInfo) => { - }); - `, - }); - // We did not run tests, so we should only have 1 skipped test. - expect(result.skipped).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(0); - expect(result.exitCode).toBe(1); - expect(result.output).not.toContain('teardown='); -}); - -test('globalSetup should work with sync function', async ({ runInlineTest }) => { - const { passed } = await runInlineTest({ - 'playwright.config.ts': ` - import * as path from 'path'; - module.exports = { - globalSetup: './globalSetup.ts', - }; - `, - 'globalSetup.ts': ` - module.exports = () => { - process.env.FOO = JSON.stringify({ foo: 'bar' }); - }; - `, - 'a.test.js': ` - const { test } = pwt; - test('should work', async ({}) => { - const value = JSON.parse(process.env.FOO); - expect(value).toEqual({ foo: 'bar' }); - }); - `, - }); - expect(passed).toBe(1); -}); - -test('globalSetup should throw when passed non-function', async ({ runInlineTest }) => { - const { output } = await runInlineTest({ - 'playwright.config.ts': ` - import * as path from 'path'; - module.exports = { - globalSetup: './globalSetup.ts', - }; - `, - 'globalSetup.ts': ` - module.exports = 42; - `, - 'a.test.js': ` - const { test } = pwt; - test('should work', async ({}) => { - }); - `, - }); - expect(output).toContain(`globalSetup.ts: globalSetup file must export a single function.`); -}); - -test('globalSetup should work with default export and run the returned fn', async ({ runInlineTest }) => { - const { output, exitCode, passed } = await runInlineTest({ - 'playwright.config.ts': ` - import * as path from 'path'; - module.exports = { - globalSetup: './globalSetup.ts', - }; - `, - 'globalSetup.ts': ` - function setup() { - let x = 42; - console.log('\\n%%setup: ' + x); - return async () => { - await x; - console.log('\\n%%teardown: ' + x); - }; - } - export default setup; - `, - 'a.test.js': ` - const { test } = pwt; - test('should work', async ({}) => { - }); - `, - }); - expect(passed).toBe(1); - expect(exitCode).toBe(0); - expect(output).toContain(`%%setup: 42`); - expect(output).toContain(`%%teardown: 42`); -}); - -test('globalSetup should allow requiring a package from node_modules', async ({ runInlineTest }) => { - const { results } = await runInlineTest({ - 'playwright.config.ts': ` - import * as path from 'path'; - module.exports = { - globalSetup: 'my-global-setup' - }; - `, - 'node_modules/my-global-setup/index.js': ` - module.exports = async () => { - await new Promise(f => setTimeout(f, 100)); - global.value = 42; - process.env.FOO = String(global.value); - }; - `, - 'a.test.js': ` - const { test } = pwt; - test('should work', async ({}, testInfo) => { - expect(process.env.FOO).toBe('42'); - }); - `, - }); - expect(results[0].status).toBe('passed'); -}); - -const authFiles = { - 'playwright.config.ts': ` - const config: pwt.PlaywrightTestConfig = { - globalSetup: require.resolve('./auth'), - use: { - baseURL: 'https://www.example.com', - storageState: 'state.json', - }, - }; - export default config; - `, - 'auth.ts': ` - async function globalSetup(config: pwt.FullConfig) { - const { baseURL, storageState } = config.projects[0].use; - const browser = await pwt.chromium.launch(); - const page = await browser.newPage(); - await page.route('**/*', route => { - route.fulfill({ body: '' }).catch(() => {}); - }); - await page.goto(baseURL!); - await page.evaluate(() => { - localStorage['name'] = 'value'; - }); - await page.context().storageState({ path: storageState as string }); - await browser.close(); - }; - export default globalSetup; - `, - 'a.test.ts': ` - const { test } = pwt; - test('should have storage state', async ({ page }) => { - await page.route('**/*', route => { - route.fulfill({ body: '' }).catch(() => {}); - }); - await page.goto('/'); - const value = await page.evaluate(() => localStorage['name']); - expect(value).toBe('value'); + test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => { + const { results, output } = await runInlineTest({ + 'playwright.config.ts': ` + import * as path from 'path'; + module.exports = { + globalSetup: './globalSetup', + globalTeardown: path.join(__dirname, 'globalTeardown.ts'), + }; + `, + 'globalSetup.ts': ` + module.exports = async () => { + await new Promise(f => setTimeout(f, 100)); + global.value = 42; + process.env.FOO = String(global.value); + }; + `, + 'globalTeardown.ts': ` + module.exports = async () => { + console.log('teardown=' + global.value); + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('should work', async ({}, testInfo) => { + expect(process.env.FOO).toBe('42'); + }); + `, + }, undefined, env); + expect(results[0].status).toBe('passed'); + expect(output).toContain('teardown=42'); }); - `, -}; -test('globalSetup should work for auth', async ({ runInlineTest }) => { - const result = await runInlineTest(authFiles); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); -}); + test('globalTeardown runs after failures', async ({ runInlineTest }) => { + const { results, output } = await runInlineTest({ + 'playwright.config.ts': ` + import * as path from 'path'; + module.exports = { + globalSetup: 'globalSetup.ts', + globalTeardown: './globalTeardown.ts', + }; + `, + 'globalSetup.ts': ` + module.exports = async () => { + await new Promise(f => setTimeout(f, 100)); + global.value = 42; + process.env.FOO = String(global.value); + }; + `, + 'globalTeardown.ts': ` + module.exports = async () => { + console.log('teardown=' + global.value); + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('should work', async ({}, testInfo) => { + expect(process.env.FOO).toBe('43'); + }); + `, + }, undefined, env); + expect(results[0].status).toBe('failed'); + expect(output).toContain('teardown=42'); + }); -test('globalSetup auth should compile', async ({ runTSC }) => { - const result = await runTSC(authFiles); - expect(result.exitCode).toBe(0); -}); + test('globalTeardown does not run when globalSetup times out', async ({ runInlineTest }) => { + test.skip(!!env.PW_TEST_LEGACY_GLOBAL_SETUP_MODE); + const result = await runInlineTest({ + 'playwright.config.ts': ` + import * as path from 'path'; + module.exports = { + globalSetup: './globalSetup.ts', + globalTeardown: 'globalTeardown.ts', + globalTimeout: 1000, + }; + `, + 'globalSetup.ts': ` + module.exports = async () => { + await new Promise(f => setTimeout(f, 10000)); + }; + `, + 'globalTeardown.ts': ` + module.exports = async () => { + console.log('teardown='); + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('should not run', async ({}, testInfo) => { + }); + `, + }, undefined, env); + // We did not run tests, so we should only have 1 skipped test. + expect(result.skipped).toBe(1); + expect(result.passed).toBe(0); + expect(result.failed).toBe(0); + expect(result.exitCode).toBe(1); + expect(result.output).not.toContain('teardown='); + }); + + test('globalSetup should work with sync function', async ({ runInlineTest }) => { + const { passed } = await runInlineTest({ + 'playwright.config.ts': ` + import * as path from 'path'; + module.exports = { + globalSetup: './globalSetup.ts', + }; + `, + 'globalSetup.ts': ` + module.exports = () => { + process.env.FOO = JSON.stringify({ foo: 'bar' }); + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('should work', async ({}) => { + const value = JSON.parse(process.env.FOO); + expect(value).toEqual({ foo: 'bar' }); + }); + `, + }, undefined, env); + expect(passed).toBe(1); + }); + + test('globalSetup should throw when passed non-function', async ({ runInlineTest }) => { + const { output } = await runInlineTest({ + 'playwright.config.ts': ` + import * as path from 'path'; + module.exports = { + globalSetup: './globalSetup.ts', + }; + `, + 'globalSetup.ts': ` + module.exports = 42; + `, + 'a.test.js': ` + const { test } = pwt; + test('should work', async ({}) => { + }); + `, + }, undefined, env); + expect(output).toContain(`globalSetup.ts: globalSetup file must export a single function.`); + }); + + test('globalSetup should work with default export and run the returned fn', async ({ runInlineTest }) => { + const { output, exitCode, passed } = await runInlineTest({ + 'playwright.config.ts': ` + import * as path from 'path'; + module.exports = { + globalSetup: './globalSetup.ts', + }; + `, + 'globalSetup.ts': ` + function setup() { + let x = 42; + console.log('\\n%%setup: ' + x); + return async () => { + await x; + console.log('\\n%%teardown: ' + x); + }; + } + export default setup; + `, + 'a.test.js': ` + const { test } = pwt; + test('should work', async ({}) => { + }); + `, + }, undefined, env); + expect(passed).toBe(1); + expect(exitCode).toBe(0); + expect(output).toContain(`%%setup: 42`); + expect(output).toContain(`%%teardown: 42`); + }); + + test('globalSetup should allow requiring a package from node_modules', async ({ runInlineTest }) => { + const { results } = await runInlineTest({ + 'playwright.config.ts': ` + import * as path from 'path'; + module.exports = { + globalSetup: 'my-global-setup' + }; + `, + 'node_modules/my-global-setup/index.js': ` + module.exports = async () => { + await new Promise(f => setTimeout(f, 100)); + global.value = 42; + process.env.FOO = String(global.value); + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('should work', async ({}, testInfo) => { + expect(process.env.FOO).toBe('42'); + }); + `, + }, undefined, env); + expect(results[0].status).toBe('passed'); + }); + + const authFiles = { + 'playwright.config.ts': ` + const config: pwt.PlaywrightTestConfig = { + globalSetup: require.resolve('./auth'), + use: { + baseURL: 'https://www.example.com', + storageState: 'state.json', + }, + }; + export default config; + `, + 'auth.ts': ` + async function globalSetup(config: pwt.FullConfig) { + const { baseURL, storageState } = config.projects[0].use; + const browser = await pwt.chromium.launch(); + const page = await browser.newPage(); + await page.route('**/*', route => { + route.fulfill({ body: '' }).catch(() => {}); + }); + await page.goto(baseURL!); + await page.evaluate(() => { + localStorage['name'] = 'value'; + }); + await page.context().storageState({ path: storageState as string }); + await browser.close(); + }; + export default globalSetup; + `, + 'a.test.ts': ` + const { test } = pwt; + test('should have storage state', async ({ page }) => { + await page.route('**/*', route => { + route.fulfill({ body: '' }).catch(() => {}); + }); + await page.goto('/'); + const value = await page.evaluate(() => localStorage['name']); + expect(value).toBe('value'); + }); + `, + }; + + test('globalSetup should work for auth', async ({ runInlineTest }) => { + const result = await runInlineTest(authFiles, undefined, env); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + }); + + test('globalSetup auth should compile', async ({ runTSC }) => { + const result = await runTSC(authFiles); + expect(result.exitCode).toBe(0); + }); + }); +}