diff --git a/.gitignore b/.gitignore index 547c965244..9d7c8736c1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,4 @@ drivers/ .gradle/ nohup.out .trace -.tmp \ No newline at end of file +.tmp diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index d78d836f31..b739d64a40 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -138,6 +138,10 @@ Output chunk. Test that was running. Note that output may happen when to test is running, in which case this will be [void]. +### param: Reporter.onStdErr.result +- `result` <[void]|[TestResult]> + +Result of the test run, this object gets populated while the test runs. ## method: Reporter.onStdOut @@ -154,7 +158,48 @@ Output chunk. Test that was running. Note that output may happen when to test is running, in which case this will be [void]. +### param: Reporter.onStdOut.result +- `result` <[void]|[TestResult]> +Result of the test run, this object gets populated while the test runs. + +## method: Reporter.onStepBegin + +Called when a test step started in the worker process. + +### param: Reporter.onStepBegin.test +- `test` <[TestCase]> + +Test that has been started. + +### param: Reporter.onStepBegin.result +- `result` <[TestResult]> + +Result of the test run, this object gets populated while the test runs. + +### param: Reporter.onStepBegin.step +- `result` <[TestStep]> + +Test step instance. + +## method: Reporter.onStepEnd + +Called when a test step finished in the worker process. + +### param: Reporter.onStepEnd.test +- `test` <[TestCase]> + +Test that has been finished. + +### param: Reporter.onStepEnd.result +- `result` <[TestResult]> + +Result of the test run. + +### param: Reporter.onStepEnd.step +- `result` <[TestStep]> + +Test step instance. ## method: Reporter.onTestBegin @@ -165,6 +210,10 @@ Called after a test has been started in the worker process. Test that has been started. +### param: Reporter.onTestBegin.result +- `result` <[TestResult]> + +Result of the test run, this object gets populated while the test runs. ## method: Reporter.onTestEnd diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md index 50473f2b5b..e0d4f870ac 100644 --- a/docs/src/test-reporter-api/class-testresult.md +++ b/docs/src/test-reporter-api/class-testresult.md @@ -49,6 +49,11 @@ Anything written to the standard error during the test run. Anything written to the standard output during the test run. +## property: TestResult.steps +- type: <[Array]<[TestStep]>> + +List of steps inside this test run. + ## property: TestResult.workerIndex - type: <[int]> diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md new file mode 100644 index 0000000000..da6bed675e --- /dev/null +++ b/docs/src/test-reporter-api/class-teststep.md @@ -0,0 +1,32 @@ +# class: TestStep +* langs: js + +Represents a step in the [TestRun]. + +## property: TestStep.category +- type: <[string]> + +Step category to differentiate steps with different origin and verbosity. Built-in categories are: +* `hook` for fixtures and hooks initialization and teardown +* `expect` for expect calls +* `pw:api` for Playwright API calls. + +## property: TestStep.duration +- type: <[float]> + +Running time in milliseconds. + +## property: TestStep.error +- type: <[void]|[TestError]> + +An error thrown during the step execution, if any. + +## property: TestStep.startTime +- type: <[Date]> + +Start time of this particular test step. + +## property: TestStep.title +- type: <[string]> + +User-friendly test step title. diff --git a/src/client/channelOwner.ts b/src/client/channelOwner.ts index ec50c2f45a..13790b15e3 100644 --- a/src/client/channelOwner.ts +++ b/src/client/channelOwner.ts @@ -23,8 +23,6 @@ import { isUnderTest } from '../utils/utils'; import type { Connection } from './connection'; import type { ClientSideInstrumentation, Logger } from './types'; -let lastCallSeq = 0; - export abstract class ChannelOwner extends EventEmitter { protected _connection: Connection; private _parent: ChannelOwner | undefined; @@ -97,19 +95,19 @@ export abstract class ChannelOwner void) | undefined; try { logApiCall(logger, `=> ${apiName} started`); - this._csi?.onApiCall({ phase: 'begin', seq, apiName, frames: stackTrace.frames }); + csiCallback = this._csi?.onApiCall(apiName); const result = await func(channel as any, stackTrace); - this._csi?.onApiCall({ phase: 'end', seq }); + csiCallback?.(); logApiCall(logger, `<= ${apiName} succeeded`); return result; } catch (e) { const innerError = ((process.env.PWDEBUGIMPL || isUnderTest()) && e.stack) ? '\n\n' + e.stack : ''; e.message = apiName + ': ' + e.message; e.stack = e.message + '\n' + frameTexts.join('\n') + innerError; - this._csi?.onApiCall({ phase: 'end', seq, error: e.stack }); + csiCallback?.(e); logApiCall(logger, `<= ${apiName} failed`); throw e; } diff --git a/src/client/types.ts b/src/client/types.ts index b0958586d7..dc4020f373 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -24,7 +24,7 @@ export interface Logger { } export interface ClientSideInstrumentation { - onApiCall(data: { phase: 'begin' | 'end', seq: number, apiName?: string, frames?: channels.StackFrame[], error?: string }): void; + onApiCall(name: string): (error?: Error) => void; } import { Size } from '../common/types'; diff --git a/src/test/dispatcher.ts b/src/test/dispatcher.ts index 3d0c0eb906..87a91aa5e5 100644 --- a/src/test/dispatcher.ts +++ b/src/test/dispatcher.ts @@ -17,8 +17,8 @@ import child_process from 'child_process'; import path from 'path'; import { EventEmitter } from 'events'; -import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, ProgressPayload } from './ipc'; -import type { TestResult, Reporter } from '../../types/testReporter'; +import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc'; +import type { TestResult, Reporter, TestStep } from '../../types/testReporter'; import { TestCase } from './test'; import { Loader } from './loader'; @@ -35,7 +35,7 @@ export class Dispatcher { private _freeWorkers: Worker[] = []; private _workerClaimers: (() => void)[] = []; - private _testById = new Map(); + private _testById = new Map }>(); private _queue: TestGroup[] = []; private _stopCallback = () => {}; readonly _loader: Loader; @@ -51,7 +51,8 @@ export class Dispatcher { for (const group of testGroups) { for (const test of group.tests) { const result = test._appendTestResult(); - this._testById.set(test._id, { test, result }); + // When changing this line, change the one in retry too. + this._testById.set(test._id, { test, result, steps: new Map() }); } } } @@ -136,7 +137,7 @@ export class Dispatcher { break; // There might be a single test that has started but has not finished yet. if (test._id !== lastStartedTestId) - this._reporter.onTestBegin?.(test); + this._reporter.onTestBegin?.(test, result); result.error = params.fatalError; result.status = first ? 'failed' : 'skipped'; this._reportTestEnd(test, result); @@ -155,6 +156,7 @@ export class Dispatcher { const pair = this._testById.get(testId)!; if (!this._isStopped && pair.test.expectedStatus === 'passed' && pair.test.results.length < pair.test.retries + 1) { pair.result = pair.test._appendTestResult(); + pair.steps = new Map(); remaining.unshift(pair.test); } } @@ -215,7 +217,7 @@ export class Dispatcher { const { test, result: testRun } = this._testById.get(params.testId)!; testRun.workerIndex = params.workerIndex; testRun.startTime = new Date(params.startWallTime); - this._reporter.onTestBegin?.(test); + this._reporter.onTestBegin?.(test, testRun); }); worker.on('testEnd', (params: TestEndPayload) => { if (this._hasReachedMaxFailures()) @@ -235,23 +237,40 @@ export class Dispatcher { test.timeout = params.timeout; this._reportTestEnd(test, result); }); - worker.on('progress', (params: ProgressPayload) => { - const { test } = this._testById.get(params.testId)!; - (this._reporter as any)._onTestProgress?.(test, params.name, params.data); + worker.on('stepBegin', (params: StepBeginPayload) => { + const { test, result, steps } = this._testById.get(params.testId)!; + const step: TestStep = { + title: params.title, + category: params.category, + startTime: new Date(params.wallTime), + duration: 0, + }; + steps.set(params.stepId, step); + result.steps.push(step); + this._reporter.onStepBegin?.(test, result, step); + }); + worker.on('stepEnd', (params: StepEndPayload) => { + const { test, result, steps } = this._testById.get(params.testId)!; + const step = steps.get(params.stepId)!; + step.duration = params.wallTime - step.startTime.getTime(); + if (params.error) + step.error = params.error; + steps.delete(params.stepId); + this._reporter.onStepEnd?.(test, result, step); }); worker.on('stdOut', (params: TestOutputPayload) => { const chunk = chunkFromParams(params); const pair = params.testId ? this._testById.get(params.testId) : undefined; if (pair) pair.result.stdout.push(chunk); - this._reporter.onStdOut?.(chunk, pair ? pair.test : undefined); + this._reporter.onStdOut?.(chunk, pair?.test, pair?.result); }); worker.on('stdErr', (params: TestOutputPayload) => { const chunk = chunkFromParams(params); const pair = params.testId ? this._testById.get(params.testId) : undefined; if (pair) pair.result.stderr.push(chunk); - this._reporter.onStdErr?.(chunk, pair ? pair.test : undefined); + this._reporter.onStdErr?.(chunk, pair?.test, pair?.result); }); worker.on('teardownError', ({error}) => { this._hasWorkerErrors = true; diff --git a/src/test/expect.ts b/src/test/expect.ts index 3d99b03ddb..166ea75459 100644 --- a/src/test/expect.ts +++ b/src/test/expect.ts @@ -39,7 +39,7 @@ import { toHaveValue } from './matchers/matchers'; import { toMatchSnapshot } from './matchers/toMatchSnapshot'; -import type { Expect } from './types'; +import type { Expect, TestStatus } from './types'; import matchers from 'expect/build/matchers'; import { currentTestInfo } from './globals'; @@ -70,41 +70,37 @@ const customMatchers = { toMatchSnapshot, }; -let lastExpectSeq = 0; - function wrap(matcherName: string, matcher: any) { return function(this: any, ...args: any[]) { const testInfo = currentTestInfo(); if (!testInfo) return matcher.call(this, ...args); - const seq = ++lastExpectSeq; - testInfo._progress('expect', { phase: 'begin', seq, matcherName }); - const endPayload: any = { phase: 'end', seq }; - let isAsync = false; + const infix = this.isNot ? '.not' : ''; + const completeStep = testInfo._addStep('expect', `expect${infix}.${matcherName}`); + + const reportStepEnd = (result: any) => { + status = result.pass !== this.isNot ? 'passed' : 'failed'; + let error: Error | undefined; + if (status === 'failed') + error = new Error(result.message()); + completeStep?.(error); + return result; + }; + + const reportStepError = (error: Error) => { + completeStep?.(error); + throw error; + }; + + let status: TestStatus = 'passed'; try { const result = matcher.call(this, ...args); - endPayload.pass = result.pass; - if (this.isNot) - endPayload.isNot = this.isNot; - if (result.pass === this.isNot && result.message) - endPayload.message = result.message(); - if (result instanceof Promise) { - isAsync = true; - return result.catch(e => { - endPayload.error = e.stack; - throw e; - }).finally(() => { - testInfo._progress('expect', endPayload); - }); - } - return result; + if (result instanceof Promise) + return result.then(reportStepEnd).catch(reportStepError); + return reportStepEnd(result); } catch (e) { - endPayload.error = e.stack; - throw e; - } finally { - if (!isAsync) - testInfo._progress('expect', endPayload); + reportStepError(e); } }; } diff --git a/src/test/index.ts b/src/test/index.ts index bb05acd72e..e963939224 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -185,7 +185,9 @@ export const test = _baseTest.extend (testInfo as any)._progress('pw:api', data), + onApiCall: (name: string) => { + return (testInfo as any)._addStep('pw:api', name); + }, }; context.setDefaultTimeout(actionTimeout || 0); context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0); diff --git a/src/test/ipc.ts b/src/test/ipc.ts index d0d48f9039..ccdb7e7ca9 100644 --- a/src/test/ipc.ts +++ b/src/test/ipc.ts @@ -46,10 +46,19 @@ export type TestEndPayload = { attachments: { name: string, path?: string, body?: string, contentType: string }[]; }; -export type ProgressPayload = { +export type StepBeginPayload = { testId: string; - name: string; - data: any; + stepId: string; + title: string; + category: string; + wallTime: number; // milliseconds since unix epoch +}; + +export type StepEndPayload = { + testId: string; + stepId: string; + wallTime: number; // milliseconds since unix epoch + error?: TestError; }; export type TestEntry = { diff --git a/src/test/reporters/multiplexer.ts b/src/test/reporters/multiplexer.ts index aafdf5b7d1..1af69a1b71 100644 --- a/src/test/reporters/multiplexer.ts +++ b/src/test/reporters/multiplexer.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { FullConfig, Suite, TestCase, TestError, TestResult, Reporter, FullResult } from '../../../types/testReporter'; +import { FullConfig, Suite, TestCase, TestError, TestResult, Reporter, FullResult, TestStep } from '../../../types/testReporter'; export class Multiplexer implements Reporter { private _reporters: Reporter[]; @@ -28,19 +28,19 @@ export class Multiplexer implements Reporter { reporter.onBegin?.(config, suite); } - onTestBegin(test: TestCase) { + onTestBegin(test: TestCase, result: TestResult) { for (const reporter of this._reporters) - reporter.onTestBegin?.(test); + reporter.onTestBegin?.(test, result); } - onStdOut(chunk: string | Buffer, test?: TestCase) { + onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { for (const reporter of this._reporters) - reporter.onStdOut?.(chunk, test); + reporter.onStdOut?.(chunk, test, result); } - onStdErr(chunk: string | Buffer, test?: TestCase) { + onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) { for (const reporter of this._reporters) - reporter.onStdErr?.(chunk, test); + reporter.onStdErr?.(chunk, test, result); } onTestEnd(test: TestCase, result: TestResult) { @@ -58,8 +58,13 @@ export class Multiplexer implements Reporter { reporter.onError?.(error); } - _onTestProgress(test: TestCase, name: string, data: any) { + onStepBegin(test: TestCase, result: TestResult, step: TestStep) { for (const reporter of this._reporters) - (reporter as any)._onTestProgress?.(test, name, data); + (reporter as any).onStepBegin?.(test, result, step); + } + + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + for (const reporter of this._reporters) + (reporter as any).onStepEnd?.(test, result, step); } } diff --git a/src/test/test.ts b/src/test/test.ts index c7bb409016..ca7e8cadd1 100644 --- a/src/test/test.ts +++ b/src/test/test.ts @@ -173,6 +173,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { stderr: [], attachments: [], status: 'skipped', + steps: [] }; this.results.push(result); return result; diff --git a/src/test/types.ts b/src/test/types.ts index ac921169a2..0a123538f4 100644 --- a/src/test/types.ts +++ b/src/test/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { Fixtures, TestInfo } from '../../types/test'; +import type { Fixtures, TestError, TestInfo } from '../../types/test'; import type { Location } from '../../types/testReporter'; export * from '../../types/test'; export { Location } from '../../types/testReporter'; @@ -25,7 +25,9 @@ export type FixturesWithLocation = { }; export type Annotations = { type: string, description?: string }[]; +export type CompleteStepCallback = (error?: TestError) => void; + export interface TestInfoImpl extends TestInfo { _testFinished: Promise; - _progress: (name: string, params: any) => void; + _addStep: (category: string, title: string) => CompleteStepCallback; } diff --git a/src/test/worker.ts b/src/test/worker.ts index 86f532737c..f63ca5ced5 100644 --- a/src/test/worker.ts +++ b/src/test/worker.ts @@ -74,7 +74,7 @@ process.on('message', async message => { workerIndex = initParams.workerIndex; startProfiling(); workerRunner = new WorkerRunner(initParams); - for (const event of ['testBegin', 'testEnd', 'done', 'progress']) + for (const event of ['testBegin', 'testEnd', 'stepBegin', 'stepEnd', 'done']) workerRunner.on(event, sendMessageToParent.bind(null, event)); return; } diff --git a/src/test/workerRunner.ts b/src/test/workerRunner.ts index 93f7551ef7..575025a07a 100644 --- a/src/test/workerRunner.ts +++ b/src/test/workerRunner.ts @@ -20,11 +20,11 @@ import rimraf from 'rimraf'; import util from 'util'; import { EventEmitter } from 'events'; import { monotonicTime, DeadlineRunner, raceAgainstDeadline, serializeError } from './util'; -import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams } from './ipc'; +import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc'; import { setCurrentTestInfo } from './globals'; import { Loader } from './loader'; import { Modifier, Suite, TestCase } from './test'; -import { Annotations, TestError, TestInfo, TestInfoImpl, WorkerInfo } from './types'; +import { Annotations, CompleteStepCallback, TestError, TestInfo, TestInfoImpl, WorkerInfo } from './types'; import { ProjectImpl } from './project'; import { FixturePool, FixtureRunner } from './fixtures'; @@ -221,6 +221,7 @@ export class WorkerRunner extends EventEmitter { })(); let testFinishedCallback = () => {}; + let lastStepId = 0; const testInfo: TestInfoImpl = { ...this._workerInfo, title: test.title, @@ -267,7 +268,26 @@ export class WorkerRunner extends EventEmitter { deadlineRunner.setDeadline(deadline()); }, _testFinished: new Promise(f => testFinishedCallback = f), - _progress: (name, data) => this.emit('progress', { testId, name, data }), + _addStep: (category: string, title: string) => { + const stepId = `${category}@${++lastStepId}`; + const payload: StepBeginPayload = { + testId, + stepId, + category, + title, + wallTime: Date.now() + }; + this.emit('stepBegin', payload); + return (error?: TestError) => { + const payload: StepEndPayload = { + testId, + stepId, + wallTime: Date.now(), + error + }; + this.emit('stepEnd', payload); + }; + }, }; // Inherit test.setTimeout() from parent suites. @@ -361,7 +381,8 @@ export class WorkerRunner extends EventEmitter { setCurrentTestInfo(currentTest ? currentTest.testInfo : null); } - private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfo) { + private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfoImpl) { + let completeStep: CompleteStepCallback | undefined; try { const beforeEachModifiers: Modifier[] = []; for (let s = test.parent; s; s = s.parent) { @@ -375,6 +396,7 @@ export class WorkerRunner extends EventEmitter { const result = await this._fixtureRunner.resolveParametersAndRunHookOrTest(modifier.fn, 'test', testInfo); testInfo[modifier.type](!!result, modifier.description!); } + completeStep = testInfo._addStep('hook', 'Before Hooks'); await this._runHooks(test.parent!, 'beforeEach', testInfo); } catch (error) { if (error instanceof SkipError) { @@ -386,6 +408,7 @@ export class WorkerRunner extends EventEmitter { } // Continue running afterEach hooks even after the failure. } + completeStep?.(testInfo.error); // Do not run the test when beforeEach hook fails. if (this._isStopped || testInfo.status === 'failed' || testInfo.status === 'skipped') @@ -409,8 +432,11 @@ export class WorkerRunner extends EventEmitter { } } - private async _runAfterHooks(test: TestCase, testInfo: TestInfo) { + private async _runAfterHooks(test: TestCase, testInfo: TestInfoImpl) { + let completeStep: CompleteStepCallback | undefined; + let teardownError: TestError | undefined; try { + completeStep = testInfo._addStep('hook', 'After Hooks'); await this._runHooks(test.parent!, 'afterEach', testInfo); } catch (error) { if (!(error instanceof SkipError)) { @@ -428,9 +454,12 @@ export class WorkerRunner extends EventEmitter { if (testInfo.status === 'passed') testInfo.status = 'failed'; // Do not overwrite test failure error. - if (!('error' in testInfo)) + if (!('error' in testInfo)) { testInfo.error = serializeError(error); + teardownError = testInfo.error; + } } + completeStep?.(teardownError); } private async _runHooks(suite: Suite, type: 'beforeEach' | 'afterEach', testInfo: TestInfo) { diff --git a/tests/config/browserTest.ts b/tests/config/browserTest.ts index 383da47034..ed494e7f7e 100644 --- a/tests/config/browserTest.ts +++ b/tests/config/browserTest.ts @@ -135,7 +135,7 @@ export const playwrightFixtures: Fixtures { + contextFactory: async ({ browser, contextOptions }, run, testInfo) => { const contexts: BrowserContext[] = []; await run(async options => { const context = await browser.newContext({ ...contextOptions, ...options }); diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 233b037d78..0883b8fef0 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -159,13 +159,16 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => { ]); }); -test('should report expect progress', async ({ runInlineTest }) => { +test('should report expect steps', async ({ runInlineTest }) => { const expectReporterJS = ` class Reporter { - _onTestProgress(test, name, data) { - if (data.frames) - data.frames = []; - console.log('%%%%', name, JSON.stringify(data)); + onStepBegin(test, result, step) { + const copy = { ...step, startTime: undefined, duration: undefined }; + console.log('%%%% begin', JSON.stringify(copy)); + } + onStepEnd(test, result, step) { + const copy = { ...step, startTime: undefined, duration: undefined }; + console.log('%%%% end', JSON.stringify(copy)); } } module.exports = Reporter; @@ -195,32 +198,45 @@ test('should report expect progress', async ({ runInlineTest }) => { expect(result.exitCode).toBe(1); expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ - `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`, - `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":true}`, - `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toBeTruthy\"}`, - `%% expect {\"phase\":\"end\",\"seq\":2,\"pass\":false,\"message\":\"\\u001b[2mexpect(\\u001b[22m\\u001b[31mreceived\\u001b[39m\\u001b[2m).\\u001b[22mtoBeTruthy\\u001b[2m()\\u001b[22m\\n\\nReceived: \\u001b[31mfalse\\u001b[39m\"}`, - - `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`, - `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":false,\"isNot\":true}`, - - `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`, - `%% pw:api {\"phase\":\"end\",\"seq\":3}`, - `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toHaveTitle\"}`, - `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.title\",\"frames\":[]}`, - `%% pw:api {\"phase\":\"end\",\"seq\":4}`, - `%% expect {\"phase\":\"end\",\"seq\":2,\"isNot\":true}`, - `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"browserContext.close\",\"frames\":[]}`, - `%% pw:api {\"phase\":\"end\",\"seq\":5}`, + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, + `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, + `%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, + `%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{}}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`, + `%% end {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `%% begin {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`, + `%% begin {\"title\":\"page.title\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"page.title\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, ]); }); -test('should report log progress', async ({ runInlineTest }) => { +test('should report api steps', async ({ runInlineTest }) => { const expectReporterJS = ` class Reporter { - _onTestProgress(test, name, data) { - if (data.frames) - data.frames = []; - console.log('%%%%', name, JSON.stringify(data)); + onStepBegin(test, result, step) { + const copy = { ...step, startTime: undefined, duration: undefined }; + console.log('%%%% begin', JSON.stringify(copy)); + } + onStepEnd(test, result, step) { + const copy = { ...step, startTime: undefined, duration: undefined }; + console.log('%%%% end', JSON.stringify(copy)); } } module.exports = Reporter; @@ -244,13 +260,17 @@ test('should report log progress', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ - `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`, - `%% pw:api {\"phase\":\"end\",\"seq\":3}`, - `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.setContent\",\"frames\":[]}`, - `%% pw:api {\"phase\":\"end\",\"seq\":4}`, - `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"page.click\",\"frames\":[]}`, - `%% pw:api {\"phase\":\"end\",\"seq\":5}`, - `%% pw:api {\"phase\":\"begin\",\"seq\":6,\"apiName\":\"browserContext.close\",\"frames\":[]}`, - `%% pw:api {\"phase\":\"end\",\"seq\":6}`, + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `%% begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, + `%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"page.click\",\"category\":\"pw:api\"}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, ]); }); diff --git a/types/testReporter.d.ts b/types/testReporter.d.ts index 05d204b619..88bc51a7b4 100644 --- a/types/testReporter.d.ts +++ b/types/testReporter.d.ts @@ -213,6 +213,39 @@ export interface TestResult { * Anything written to the standard error during the test run. */ stderr: (string | Buffer)[]; + /** + * List of steps inside this test run. + */ + steps: TestStep[]; +} + +/** + * Represents a step in the [TestRun]. + */ +export interface TestStep { + /** + * User-friendly test step title. + */ + title: string; + /** + * Step category to differentiate steps with different origin and verbosity. Built-in categories are: + * - `hook` for fixtures and hooks initialization and teardown + * - `expect` for expect calls + * - `pw:api` for Playwright API calls. + */ + category: string, + /** + * Start time of this particular test step. + */ + startTime: Date; + /** + * Running time in milliseconds. + */ + duration: number; + /** + * An error thrown during the step execution, if any. + */ + error?: TestError; } /** @@ -321,26 +354,43 @@ export interface Reporter { /** * Called after a test has been started in the worker process. * @param test Test that has been started. + * @param result Result of the test run, this object gets populated while the test runs. */ - onTestBegin?(test: TestCase): void; + onTestBegin?(test: TestCase, result: TestResult): void; /** * Called when something has been written to the standard output in the worker process. * @param chunk Output chunk. * @param test Test that was running. Note that output may happen when to test is running, in which case this will be [void]. + * @param result Result of the test run, this object gets populated while the test runs. */ - onStdOut?(chunk: string | Buffer, test?: TestCase): void; + onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; /** * Called when something has been written to the standard error in the worker process. * @param chunk Output chunk. * @param test Test that was running. Note that output may happen when to test is running, in which case this will be [void]. + * @param result Result of the test run, this object gets populated while the test runs. */ - onStdErr?(chunk: string | Buffer, test?: TestCase): void; + onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; /** * Called after a test has been finished in the worker process. * @param test Test that has been finished. * @param result Result of the test run. */ onTestEnd?(test: TestCase, result: TestResult): void; + /** + * Called when a test step started in the worker process. + * @param test Test that has been started. + * @param result Result of the test run, this object gets populated while the test runs. + * @param result Test step instance. + */ + onStepBegin?(test: TestCase, result: TestResult, step: TestStep): void; + /** + * Called when a test step finished in the worker process. + * @param test Test that has been finished. + * @param result Result of the test run. + * @param result Test step instance. + */ + onStepEnd?(test: TestCase, result: TestResult, step: TestStep): void; /** * Called on some global error, for example unhandled exception in the worker process. * @param error The error. diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 1d070cedab..741bb7280f 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -55,6 +55,15 @@ export interface TestResult { attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; stdout: (string | Buffer)[]; stderr: (string | Buffer)[]; + steps: TestStep[]; +} + +export interface TestStep { + title: string; + category: string, + startTime: Date; + duration: number; + error?: TestError; } /** @@ -73,10 +82,12 @@ export interface FullResult { export interface Reporter { onBegin?(config: FullConfig, suite: Suite): void; - onTestBegin?(test: TestCase): void; - onStdOut?(chunk: string | Buffer, test?: TestCase): void; - onStdErr?(chunk: string | Buffer, test?: TestCase): void; + onTestBegin?(test: TestCase, result: TestResult): void; + onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; + onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; onTestEnd?(test: TestCase, result: TestResult): void; + onStepBegin?(test: TestCase, result: TestResult, step: TestStep): void; + onStepEnd?(test: TestCase, result: TestResult, step: TestStep): void; onError?(error: TestError): void; onEnd?(result: FullResult): void | Promise; }