diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index da6bed675e..9ab3b678da 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -26,6 +26,11 @@ An error thrown during the step execution, if any. Start time of this particular test step. +## property: TestStep.steps +- type: <[Array]<[TestStep]>> + +List of steps inside this step. + ## property: TestStep.title - type: <[string]> diff --git a/src/test/dispatcher.ts b/src/test/dispatcher.ts index 548ad5b398..c838b0f51a 100644 --- a/src/test/dispatcher.ts +++ b/src/test/dispatcher.ts @@ -35,7 +35,7 @@ export class Dispatcher { private _freeWorkers: Worker[] = []; private _workerClaimers: (() => void)[] = []; - private _testById = new Map }>(); + private _testById = new Map, stepStack: Set }>(); private _queue: TestGroup[] = []; private _stopCallback = () => {}; readonly _loader: Loader; @@ -52,7 +52,7 @@ export class Dispatcher { for (const test of group.tests) { const result = test._appendTestResult(); // When changing this line, change the one in retry too. - this._testById.set(test._id, { test, result, steps: new Map() }); + this._testById.set(test._id, { test, result, steps: new Map(), stepStack: new Set() }); } } } @@ -197,6 +197,7 @@ export class Dispatcher { if (!this._isStopped && pair.test.expectedStatus === 'passed' && pair.test.results.length < pair.test.retries + 1) { pair.result = pair.test._appendTestResult(); pair.steps = new Map(); + pair.stepStack = new Set(); remaining.push(pair.test); } } @@ -280,19 +281,22 @@ export class Dispatcher { this._reportTestEnd(test, result); }); worker.on('stepBegin', (params: StepBeginPayload) => { - const { test, result, steps } = this._testById.get(params.testId)!; + const { test, result, steps, stepStack } = this._testById.get(params.testId)!; const step: TestStep = { title: params.title, category: params.category, startTime: new Date(params.wallTime), duration: 0, + steps: [], }; steps.set(params.stepId, step); - result.steps.push(step); + const parentStep = [...stepStack].pop() || result; + parentStep.steps.push(step); + stepStack.add(step); this._reporter.onStepBegin?.(test, result, step); }); worker.on('stepEnd', (params: StepEndPayload) => { - const { test, result, steps } = this._testById.get(params.testId)!; + const { test, result, steps, stepStack } = this._testById.get(params.testId)!; const step = steps.get(params.stepId); if (!step) { this._reporter.onStdErr?.('Internal error: step end without step begin: ' + params.stepId, test, result); @@ -301,6 +305,7 @@ export class Dispatcher { step.duration = params.wallTime - step.startTime.getTime(); if (params.error) step.error = params.error; + stepStack.delete(step); steps.delete(params.stepId); this._reporter.onStepEnd?.(test, result, step); }); diff --git a/src/test/reporters/html.ts b/src/test/reporters/html.ts index bcc71a5854..928dd28052 100644 --- a/src/test/reporters/html.ts +++ b/src/test/reporters/html.ts @@ -199,7 +199,7 @@ class HtmlReporter { attachments: await this._createAttachments(testId, result), stdout: result.stdout, stderr: result.stderr, - steps: this._serializeSteps(result.steps) + steps: serializeSteps(result.steps) }; } @@ -249,43 +249,19 @@ class HtmlReporter { sha1 }; } - - private _serializeSteps(steps: TestStep[]): JsonTestStep[] { - const stepStack: TestStep[] = []; - const result: JsonTestStep[] = []; - const stepMap = new Map(); - for (const step of steps) { - let lastStep = stepStack[stepStack.length - 1]; - while (lastStep && !containsStep(lastStep, step)) { - stepStack.pop(); - lastStep = stepStack[stepStack.length - 1]; - } - const collection = stepMap.get(lastStep!)?.steps || result; - const jsonStep = { - title: step.title, - category: step.category, - startTime: step.startTime.toISOString(), - duration: step.duration, - error: step.error, - steps: [] - }; - collection.push(jsonStep); - stepMap.set(step, jsonStep); - stepStack.push(step); - } - return result; - } } - -function containsStep(outer: TestStep, inner: TestStep): boolean { - if (outer.startTime.getTime() > inner.startTime.getTime()) - return false; - if (outer.startTime.getTime() + outer.duration < inner.startTime.getTime() + inner.duration) - return false; - if (outer.startTime.getTime() + outer.duration <= inner.startTime.getTime()) - return false; - return true; +function serializeSteps(steps: TestStep[]): JsonTestStep[] { + return steps.map(step => { + return { + title: step.title, + category: step.category, + startTime: step.startTime.toISOString(), + duration: step.duration, + error: step.error, + steps: serializeSteps(step.steps), + }; + }); } function isTextAttachment(contentType: string) { diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index ef8dde5183..66313c1f12 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -165,11 +165,11 @@ test('should report expect steps', async ({ runInlineTest }) => { process.stdout.write(chunk); } onStepBegin(test, result, step) { - const copy = { ...step, startTime: undefined, duration: undefined }; + const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined }; console.log('%%%% begin', JSON.stringify(copy)); } onStepEnd(test, result, step) { - const copy = { ...step, startTime: undefined, duration: undefined }; + const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined }; if (copy.error?.stack) copy.error.stack = ''; console.log('%%%% end', JSON.stringify(copy)); @@ -244,11 +244,11 @@ test('should report api steps', async ({ runInlineTest }) => { console.log('%%%% test end ' + test.title); } onStepBegin(test, result, step) { - const copy = { ...step, startTime: undefined, duration: undefined }; + const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined }; console.log('%%%% begin', JSON.stringify(copy)); } onStepEnd(test, result, step) { - const copy = { ...step, startTime: undefined, duration: undefined }; + const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined }; if (copy.error?.stack) copy.error.stack = ''; console.log('%%%% end', JSON.stringify(copy)); @@ -335,11 +335,11 @@ test('should report api step failure', async ({ runInlineTest }) => { process.stdout.write(chunk); } onStepBegin(test, result, step) { - const copy = { ...step, startTime: undefined, duration: undefined }; + const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined }; console.log('%%%% begin', JSON.stringify(copy)); } onStepEnd(test, result, step) { - const copy = { ...step, startTime: undefined, duration: undefined }; + const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined }; if (copy.error?.stack) copy.error.stack = ''; console.log('%%%% end', JSON.stringify(copy)); @@ -388,11 +388,11 @@ test('should report test.step', async ({ runInlineTest }) => { process.stdout.write(chunk); } onStepBegin(test, result, step) { - const copy = { ...step, startTime: undefined, duration: undefined }; + const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined }; console.log('%%%% begin', JSON.stringify(copy)); } onStepEnd(test, result, step) { - const copy = { ...step, startTime: undefined, duration: undefined }; + const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined }; if (copy.error?.stack) copy.error.stack = ''; console.log('%%%% end', JSON.stringify(copy)); @@ -435,6 +435,139 @@ test('should report test.step', async ({ runInlineTest }) => { ]); }); +test('should report api step hierarchy', async ({ runInlineTest }) => { + const expectReporterJS = ` + class Reporter { + onBegin(config: FullConfig, suite: Suite) { + this.suite = suite; + } + + async onEnd() { + const processSuite = (suite: Suite) => { + for (const child of suite.suites) + processSuite(child); + for (const test of suite.tests) { + for (const result of test.results) { + for (const step of result.steps) { + console.log('%% ' + JSON.stringify(step)); + } + } + } + }; + processSuite(this.suite); + } + } + module.exports = Reporter; + `; + + const result = await runInlineTest({ + 'reporter.ts': expectReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({ page }) => { + await test.step('outer step 1', async () => { + await test.step('inner step 1.1', async () => {}); + await test.step('inner step 1.2', async () => {}); + }); + await test.step('outer step 2', async () => { + await test.step('inner step 2.1', async () => {}); + await test.step('inner step 2.2', async () => {}); + }); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + const objects = result.output.split('\n').filter(line => line.startsWith('%% ')).map(line => line.substring(3).trim()).filter(Boolean).map(line => JSON.parse(line)); + const distill = step => { + step.duration = 1; + step.startTime = 'time'; + step.steps.forEach(distill); + }; + objects.forEach(distill); + expect(objects).toEqual([ + { + category: 'hook', + title: 'Before Hooks', + duration: 1, + startTime: 'time', + steps: [ + { + category: 'pw:api', + title: 'browserContext.newPage', + duration: 1, + startTime: 'time', + steps: [], + }, + ], + }, + { + category: 'test.step', + title: 'outer step 1', + duration: 1, + startTime: 'time', + steps: [ + { + category: 'test.step', + title: 'inner step 1.1', + duration: 1, + startTime: 'time', + steps: [], + }, + { + category: 'test.step', + title: 'inner step 1.2', + duration: 1, + startTime: 'time', + steps: [], + }, + ], + }, + { + category: 'test.step', + title: 'outer step 2', + duration: 1, + startTime: 'time', + steps: [ + { + category: 'test.step', + title: 'inner step 2.1', + duration: 1, + startTime: 'time', + steps: [], + }, + { + category: 'test.step', + title: 'inner step 2.2', + duration: 1, + startTime: 'time', + steps: [], + }, + ], + }, + { + category: 'hook', + title: 'After Hooks', + duration: 1, + startTime: 'time', + steps: [ + { + category: 'pw:api', + title: 'browserContext.close', + duration: 1, + startTime: 'time', + steps: [], + }, + ], + }, + ]); +}); + function stripEscapedAscii(str: string) { return str.replace(/\\u00[a-z0-9][a-z0-9]\[[^m]+m/g, ''); } diff --git a/types/testReporter.d.ts b/types/testReporter.d.ts index d15aefe766..3e8020da88 100644 --- a/types/testReporter.d.ts +++ b/types/testReporter.d.ts @@ -246,6 +246,10 @@ export interface TestStep { * An error thrown during the step execution, if any. */ error?: TestError; + /** + * List of steps inside this step. + */ + steps: TestStep[]; } /** diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 741bb7280f..b9a05be7b9 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -64,6 +64,7 @@ export interface TestStep { startTime: Date; duration: number; error?: TestError; + steps: TestStep[]; } /**