diff --git a/src/client/channelOwner.ts b/src/client/channelOwner.ts index 90cb40e68d..ec50c2f45a 100644 --- a/src/client/channelOwner.ts +++ b/src/client/channelOwner.ts @@ -21,7 +21,9 @@ import { debugLogger } from '../utils/debugLogger'; import { captureStackTrace, ParsedStackTrace } from '../utils/stackTrace'; import { isUnderTest } from '../utils/utils'; import type { Connection } from './connection'; -import type { Logger } from './types'; +import type { ClientSideInstrumentation, Logger } from './types'; + +let lastCallSeq = 0; export abstract class ChannelOwner extends EventEmitter { protected _connection: Connection; @@ -33,6 +35,7 @@ export abstract class ChannelOwner ${apiName} started`); + this._csi?.onApiCall({ phase: 'begin', seq, apiName, frames: stackTrace.frames }); const result = await func(channel as any, stackTrace); + this._csi?.onApiCall({ phase: 'end', seq }); logApiCall(logger, `<= ${apiName} succeeded`); return result; } catch (e) { const innerError = ((process.env.PWDEBUGIMPL || isUnderTest()) && e.stack) ? '\n\n' + e.stack : ''; e.message = apiName + ': ' + e.message; e.stack = e.message + '\n' + frameTexts.join('\n') + innerError; + this._csi?.onApiCall({ phase: 'end', seq, error: e.stack }); logApiCall(logger, `<= ${apiName} failed`); throw e; } diff --git a/src/client/types.ts b/src/client/types.ts index 57ad4d1eeb..b0958586d7 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -23,6 +23,10 @@ export interface Logger { log(name: string, severity: LoggerSeverity, message: string | Error, args: any[], hints: { color?: string }): void; } +export interface ClientSideInstrumentation { + onApiCall(data: { phase: 'begin' | 'end', seq: number, apiName?: string, frames?: channels.StackFrame[], error?: string }): void; +} + import { Size } from '../common/types'; export { Size, Point, Rect, Quad, URLMatch, TimeoutOptions } from '../common/types'; export type StrictOptions = { strict?: boolean }; diff --git a/src/test/dispatcher.ts b/src/test/dispatcher.ts index bc2a17259a..3d0c0eb906 100644 --- a/src/test/dispatcher.ts +++ b/src/test/dispatcher.ts @@ -17,7 +17,7 @@ import child_process from 'child_process'; import path from 'path'; import { EventEmitter } from 'events'; -import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams } from './ipc'; +import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, ProgressPayload } from './ipc'; import type { TestResult, Reporter } from '../../types/testReporter'; import { TestCase } from './test'; import { Loader } from './loader'; @@ -235,6 +235,10 @@ export class Dispatcher { test.timeout = params.timeout; this._reportTestEnd(test, result); }); + worker.on('progress', (params: ProgressPayload) => { + const { test } = this._testById.get(params.testId)!; + (this._reporter as any)._onTestProgress?.(test, params.name, params.data); + }); worker.on('stdOut', (params: TestOutputPayload) => { const chunk = chunkFromParams(params); const pair = params.testId ? this._testById.get(params.testId) : undefined; diff --git a/src/test/expect.ts b/src/test/expect.ts index 980f499b1f..3d99b03ddb 100644 --- a/src/test/expect.ts +++ b/src/test/expect.ts @@ -40,10 +40,12 @@ import { } from './matchers/matchers'; import { toMatchSnapshot } from './matchers/toMatchSnapshot'; import type { Expect } from './types'; +import matchers from 'expect/build/matchers'; +import { currentTestInfo } from './globals'; export const expect: Expect = expectLibrary as any; expectLibrary.setState({ expand: false }); -expectLibrary.extend({ +const customMatchers = { toBeChecked, toBeDisabled, toBeEditable, @@ -66,4 +68,51 @@ expectLibrary.extend({ toHaveURL, toHaveValue, toMatchSnapshot, -}); +}; + +let lastExpectSeq = 0; + +function wrap(matcherName: string, matcher: any) { + return function(this: any, ...args: any[]) { + const testInfo = currentTestInfo(); + if (!testInfo) + return matcher.call(this, ...args); + + const seq = ++lastExpectSeq; + testInfo._progress('expect', { phase: 'begin', seq, matcherName }); + const endPayload: any = { phase: 'end', seq }; + let isAsync = false; + try { + const result = matcher.call(this, ...args); + endPayload.pass = result.pass; + if (this.isNot) + endPayload.isNot = this.isNot; + if (result.pass === this.isNot && result.message) + endPayload.message = result.message(); + if (result instanceof Promise) { + isAsync = true; + return result.catch(e => { + endPayload.error = e.stack; + throw e; + }).finally(() => { + testInfo._progress('expect', endPayload); + }); + } + return result; + } catch (e) { + endPayload.error = e.stack; + throw e; + } finally { + if (!isAsync) + testInfo._progress('expect', endPayload); + } + }; +} + +const wrappedMatchers: any = {}; +for (const matcherName in matchers) + wrappedMatchers[matcherName] = wrap(matcherName, matchers[matcherName]); +for (const matcherName in customMatchers) + wrappedMatchers[matcherName] = wrap(matcherName, (customMatchers as any)[matcherName]); + +expectLibrary.extend(wrappedMatchers); diff --git a/src/test/index.ts b/src/test/index.ts index 081d3a70b7..bb05acd72e 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -177,13 +177,16 @@ export const test = _baseTest.extend (testInfo as any)._progress('pw:api', data), + }; context.setDefaultTimeout(actionTimeout || 0); context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0); expectLibrary.setState({ playwrightActionTimeout: actionTimeout } as any); diff --git a/src/test/ipc.ts b/src/test/ipc.ts index 7f7c5768fc..d0d48f9039 100644 --- a/src/test/ipc.ts +++ b/src/test/ipc.ts @@ -46,6 +46,12 @@ export type TestEndPayload = { attachments: { name: string, path?: string, body?: string, contentType: string }[]; }; +export type ProgressPayload = { + testId: string; + name: string; + data: any; +}; + export type TestEntry = { testId: string; retry: number; diff --git a/src/test/reporters/multiplexer.ts b/src/test/reporters/multiplexer.ts index 0824d2f4c0..aafdf5b7d1 100644 --- a/src/test/reporters/multiplexer.ts +++ b/src/test/reporters/multiplexer.ts @@ -57,4 +57,9 @@ export class Multiplexer implements Reporter { for (const reporter of this._reporters) reporter.onError?.(error); } + + _onTestProgress(test: TestCase, name: string, data: any) { + for (const reporter of this._reporters) + (reporter as any)._onTestProgress?.(test, name, data); + } } diff --git a/src/test/types.ts b/src/test/types.ts index de49c9d554..ac921169a2 100644 --- a/src/test/types.ts +++ b/src/test/types.ts @@ -27,4 +27,5 @@ export type Annotations = { type: string, description?: string }[]; export interface TestInfoImpl extends TestInfo { _testFinished: Promise; + _progress: (name: string, params: any) => void; } diff --git a/src/test/worker.ts b/src/test/worker.ts index a5b8d1db02..86f532737c 100644 --- a/src/test/worker.ts +++ b/src/test/worker.ts @@ -74,7 +74,7 @@ process.on('message', async message => { workerIndex = initParams.workerIndex; startProfiling(); workerRunner = new WorkerRunner(initParams); - for (const event of ['testBegin', 'testEnd', 'done']) + for (const event of ['testBegin', 'testEnd', 'done', 'progress']) workerRunner.on(event, sendMessageToParent.bind(null, event)); return; } diff --git a/src/test/workerRunner.ts b/src/test/workerRunner.ts index a00ff9c009..93f7551ef7 100644 --- a/src/test/workerRunner.ts +++ b/src/test/workerRunner.ts @@ -267,6 +267,7 @@ export class WorkerRunner extends EventEmitter { deadlineRunner.setDeadline(deadline()); }, _testFinished: new Promise(f => testFinishedCallback = f), + _progress: (name, data) => this.emit('progress', { testId, name, data }), }; // Inherit test.setTimeout() from parent suites. diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index ef153fb88c..233b037d78 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -158,3 +158,99 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => { '%%end', ]); }); + +test('should report expect progress', async ({ runInlineTest }) => { + const expectReporterJS = ` + class Reporter { + _onTestProgress(test, name, data) { + if (data.frames) + data.frames = []; + console.log('%%%%', name, JSON.stringify(data)); + } + } + module.exports = Reporter; + `; + + const result = await runInlineTest({ + 'reporter.ts': expectReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('fail', async ({}) => { + expect(true).toBeTruthy(); + expect(false).toBeTruthy(); + }); + test('pass', async ({}) => { + expect(false).not.toBeTruthy(); + }); + test('async', async ({ page }) => { + await expect(page).not.toHaveTitle('False'); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`, + `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":true}`, + `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toBeTruthy\"}`, + `%% expect {\"phase\":\"end\",\"seq\":2,\"pass\":false,\"message\":\"\\u001b[2mexpect(\\u001b[22m\\u001b[31mreceived\\u001b[39m\\u001b[2m).\\u001b[22mtoBeTruthy\\u001b[2m()\\u001b[22m\\n\\nReceived: \\u001b[31mfalse\\u001b[39m\"}`, + + `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`, + `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":false,\"isNot\":true}`, + + `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`, + `%% pw:api {\"phase\":\"end\",\"seq\":3}`, + `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toHaveTitle\"}`, + `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.title\",\"frames\":[]}`, + `%% pw:api {\"phase\":\"end\",\"seq\":4}`, + `%% expect {\"phase\":\"end\",\"seq\":2,\"isNot\":true}`, + `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"browserContext.close\",\"frames\":[]}`, + `%% pw:api {\"phase\":\"end\",\"seq\":5}`, + ]); +}); + +test('should report log progress', async ({ runInlineTest }) => { + const expectReporterJS = ` + class Reporter { + _onTestProgress(test, name, data) { + if (data.frames) + data.frames = []; + console.log('%%%%', name, JSON.stringify(data)); + } + } + module.exports = Reporter; + `; + + const result = await runInlineTest({ + 'reporter.ts': expectReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({ page }) => { + await page.setContent(''); + await page.click('button'); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`, + `%% pw:api {\"phase\":\"end\",\"seq\":3}`, + `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.setContent\",\"frames\":[]}`, + `%% pw:api {\"phase\":\"end\",\"seq\":4}`, + `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"page.click\",\"frames\":[]}`, + `%% pw:api {\"phase\":\"end\",\"seq\":5}`, + `%% pw:api {\"phase\":\"begin\",\"seq\":6,\"apiName\":\"browserContext.close\",\"frames\":[]}`, + `%% pw:api {\"phase\":\"end\",\"seq\":6}`, + ]); +});