From 440651e05c39a4d2aee70abfc7feed49911e8119 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Mon, 13 Sep 2021 17:49:58 -0700 Subject: [PATCH] test(test runner): calling test.step() from fixture (#8899) --- tests/playwright-test/reporter.spec.ts | 147 ------------------- tests/playwright-test/test-step.spec.ts | 184 ++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 147 deletions(-) create mode 100644 tests/playwright-test/test-step.spec.ts diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 8dfc2f9646..a42eeb814e 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -337,41 +337,6 @@ test('should report api step failure', async ({ runInlineTest }) => { ]); }); -test('should report test.step', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepsReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - const { test } = pwt; - test('pass', async ({ page }) => { - await test.step('First step', async () => { - expect(1).toBe(2); - }); - }); - ` - }, { reporter: '', workers: 1 }); - - expect(result.exitCode).toBe(1); - expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([ - `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, - `%% begin {\"title\":\"First step\",\"category\":\"test.step\"}`, - `%% begin {\"title\":\"expect.toBe\",\"category\":\"expect\"}`, - `%% end {\"title\":\"expect.toBe\",\"category\":\"expect\",\"error\":{\"message\":\"expect(received).toBe(expected) // Object.is equality\\n\\nExpected: 2\\nReceived: 1\",\"stack\":\"\"}}`, - `%% end {\"title\":\"First step\",\"category\":\"test.step\",\"steps\":[{\"title\":\"expect.toBe\",\"category\":\"expect\",\"error\":{\"message\":\"expect(received).toBe(expected) // Object.is equality\\n\\nExpected: 2\\nReceived: 1\",\"stack\":\"\"}}],\"error\":{\"message\":\"expect(received).toBe(expected) // Object.is equality\\n\\nExpected: 2\\nReceived: 1\",\"stack\":\"\"}}`, - `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, - `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, - `%% end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"browserContext.close\",\"category\":\"pw:api\"}]}`, - ]); -}); - test('should not have internal error when steps are finished after timeout', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.ts': ` @@ -394,118 +359,6 @@ test('should not have internal error when steps are finished after timeout', asy expect(result.output).not.toContain('Internal error'); }); -test('should report api step hierarchy', async ({ runInlineTest }) => { - const expectReporterJS = ` - class Reporter { - onBegin(config: FullConfig, suite: Suite) { - this.suite = suite; - } - - distillStep(step) { - return { - ...step, - startTime: undefined, - duration: undefined, - parent: undefined, - data: undefined, - steps: step.steps.length ? step.steps.map(s => this.distillStep(s)) : undefined, - }; - } - - async onEnd() { - const processSuite = (suite: Suite) => { - for (const child of suite.suites) - processSuite(child); - for (const test of suite.tests) { - for (const result of test.results) { - for (const step of result.steps) { - console.log('%% ' + JSON.stringify(this.distillStep(step))); - } - } - } - }; - processSuite(this.suite); - } - } - module.exports = Reporter; - `; - - const result = await runInlineTest({ - 'reporter.ts': expectReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - const { test } = pwt; - test('pass', async ({ page }) => { - await test.step('outer step 1', async () => { - await test.step('inner step 1.1', async () => {}); - await test.step('inner step 1.2', async () => {}); - }); - await test.step('outer step 2', async () => { - await test.step('inner step 2.1', async () => {}); - await test.step('inner step 2.2', async () => {}); - }); - }); - ` - }, { reporter: '', workers: 1 }); - - expect(result.exitCode).toBe(0); - const objects = result.output.split('\n').filter(line => line.startsWith('%% ')).map(line => line.substring(3).trim()).filter(Boolean).map(line => JSON.parse(line)); - expect(objects).toEqual([ - { - category: 'hook', - title: 'Before Hooks', - steps: [ - { - category: 'pw:api', - title: 'browserContext.newPage', - }, - ], - }, - { - category: 'test.step', - title: 'outer step 1', - steps: [ - { - category: 'test.step', - title: 'inner step 1.1', - }, - { - category: 'test.step', - title: 'inner step 1.2', - }, - ], - }, - { - category: 'test.step', - title: 'outer step 2', - steps: [ - { - category: 'test.step', - title: 'inner step 2.1', - }, - { - category: 'test.step', - title: 'inner step 2.2', - }, - ], - }, - { - category: 'hook', - title: 'After Hooks', - steps: [ - { - category: 'pw:api', - title: 'browserContext.close', - }, - ], - }, - ]); -}); - test('should report expect and pw:api stacks and logs', async ({ runInlineTest }, testInfo) => { const expectReporterJS = ` class Reporter { diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.ts new file mode 100644 index 0000000000..156c9b13ef --- /dev/null +++ b/tests/playwright-test/test-step.spec.ts @@ -0,0 +1,184 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('should report api step hierarchy', async ({ runInlineTest }) => { + const expectReporterJS = ` + class Reporter { + onBegin(config: FullConfig, suite: Suite) { + this.suite = suite; + } + + distillStep(step) { + return { + ...step, + startTime: undefined, + duration: undefined, + parent: undefined, + data: undefined, + steps: step.steps.length ? step.steps.map(s => this.distillStep(s)) : undefined, + }; + } + + async onEnd() { + const processSuite = (suite: Suite) => { + for (const child of suite.suites) + processSuite(child); + for (const test of suite.tests) { + for (const result of test.results) { + for (const step of result.steps) { + console.log('%% ' + JSON.stringify(this.distillStep(step))); + } + } + } + }; + processSuite(this.suite); + } + } + module.exports = Reporter; + `; + + const result = await runInlineTest({ + 'reporter.ts': expectReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({ page }) => { + await test.step('outer step 1', async () => { + await test.step('inner step 1.1', async () => {}); + await test.step('inner step 1.2', async () => {}); + }); + await test.step('outer step 2', async () => { + await test.step('inner step 2.1', async () => {}); + await test.step('inner step 2.2', async () => {}); + }); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + const objects = result.output.split('\n').filter(line => line.startsWith('%% ')).map(line => line.substring(3).trim()).filter(Boolean).map(line => JSON.parse(line)); + expect(objects).toEqual([ + { + category: 'hook', + title: 'Before Hooks', + steps: [ + { + category: 'pw:api', + title: 'browserContext.newPage', + }, + ], + }, + { + category: 'test.step', + title: 'outer step 1', + steps: [ + { + category: 'test.step', + title: 'inner step 1.1', + }, + { + category: 'test.step', + title: 'inner step 1.2', + }, + ], + }, + { + category: 'test.step', + title: 'outer step 2', + steps: [ + { + category: 'test.step', + title: 'inner step 2.1', + }, + { + category: 'test.step', + title: 'inner step 2.2', + }, + ], + }, + { + category: 'hook', + title: 'After Hooks', + steps: [ + { + category: 'pw:api', + title: 'browserContext.close', + }, + ], + }, + ]); +}); + +test('should report test.step from fixtures', async ({ runInlineTest }) => { + const expectReporterJS = ` + class Reporter { + onStepBegin(test, result, step) { + console.log('%% begin ' + step.title); + } + onStepEnd(test, result, step) { + console.log('%% end ' + step.title); + } + } + module.exports = Reporter; + `; + + const result = await runInlineTest({ + 'reporter.ts': expectReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + const test = pwt.test.extend({ + foo: async ({}, use) => { + await pwt.test.step('setup foo', () => {}); + await use(async () => { + await test.step('inside foo', () => {}); + }); + await test.step('teardown foo', () => {}); + }, + }); + test('pass', async ({ foo }) => { + await test.step('test step', async () => { + await foo(); + }); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + `%% begin Before Hooks`, + `%% begin setup foo`, + `%% end setup foo`, + `%% end Before Hooks`, + `%% begin test step`, + `%% begin inside foo`, + `%% end inside foo`, + `%% end test step`, + `%% begin After Hooks`, + `%% begin teardown foo`, + `%% end teardown foo`, + `%% end After Hooks`, + ]); +});