feat(test-runner): experiemental expect plumbing (#7926)

src/client/channelOwner.ts

@@ -21,7 +21,9 @@ import { debugLogger } from '../utils/debugLogger';
 import { captureStackTrace, ParsedStackTrace } from '../utils/stackTrace';
 import { isUnderTest } from '../utils/utils';
 import type { Connection } from './connection';
-import type { Logger } from './types';
+import type { ClientSideInstrumentation, Logger } from './types';
+
+let lastCallSeq = 0;
 
 export abstract class ChannelOwner<T extends channels.Channel = channels.Channel, Initializer = {}> extends EventEmitter {
   protected _connection: Connection;
@@ -33,6 +35,7 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
   readonly _channel: T;
   readonly _initializer: Initializer;
   _logger: Logger | undefined;
+  _csi: ClientSideInstrumentation | undefined;
 
   constructor(parent: ChannelOwner | Connection, type: string, guid: string, initializer: Initializer) {
     super();
@@ -46,6 +49,7 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
     if (this._parent) {
       this._parent._objects.set(guid, this);
       this._logger = this._parent._logger;
+      this._csi = this._parent._csi;
     }
 
     this._channel = this._createChannel(new EventEmitter(), null);
@@ -93,15 +97,19 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
     const stackTrace = captureStackTrace();
     const { apiName, frameTexts } = stackTrace;
     const channel = this._createChannel({}, stackTrace);
+    const seq = ++lastCallSeq;
     try {
       logApiCall(logger, `=> ${apiName} started`);
+      this._csi?.onApiCall({ phase: 'begin', seq, apiName, frames: stackTrace.frames });
       const result = await func(channel as any, stackTrace);
+      this._csi?.onApiCall({ phase: 'end', seq });
       logApiCall(logger, `<= ${apiName} succeeded`);
       return result;
     } catch (e) {
       const innerError = ((process.env.PWDEBUGIMPL || isUnderTest()) && e.stack) ? '\n<inner error>\n' + e.stack : '';
       e.message = apiName + ': ' + e.message;
       e.stack = e.message + '\n' + frameTexts.join('\n') + innerError;
+      this._csi?.onApiCall({ phase: 'end', seq, error: e.stack });
       logApiCall(logger, `<= ${apiName} failed`);
       throw e;
     }

src/client/types.ts

@@ -23,6 +23,10 @@ export interface Logger {
   log(name: string, severity: LoggerSeverity, message: string | Error, args: any[], hints: { color?: string }): void;
 }
 
+export interface ClientSideInstrumentation {
+  onApiCall(data: { phase: 'begin' | 'end', seq: number, apiName?: string, frames?: channels.StackFrame[], error?: string }): void;
+}
+
 import { Size } from '../common/types';
 export { Size, Point, Rect, Quad, URLMatch, TimeoutOptions } from '../common/types';
 export type StrictOptions = { strict?: boolean };

src/test/dispatcher.ts

@@ -17,7 +17,7 @@
 import child_process from 'child_process';
 import path from 'path';
 import { EventEmitter } from 'events';
-import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams } from './ipc';
+import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, ProgressPayload } from './ipc';
 import type { TestResult, Reporter } from '../../types/testReporter';
 import { TestCase } from './test';
 import { Loader } from './loader';
@@ -235,6 +235,10 @@ export class Dispatcher {
       test.timeout = params.timeout;
       this._reportTestEnd(test, result);
     });
+    worker.on('progress', (params: ProgressPayload) => {
+      const { test } = this._testById.get(params.testId)!;
+      (this._reporter as any)._onTestProgress?.(test, params.name, params.data);
+    });
     worker.on('stdOut', (params: TestOutputPayload) => {
       const chunk = chunkFromParams(params);
       const pair = params.testId ? this._testById.get(params.testId) : undefined;

src/test/expect.ts

@@ -40,10 +40,12 @@ import {
 } from './matchers/matchers';
 import { toMatchSnapshot } from './matchers/toMatchSnapshot';
 import type { Expect } from './types';
+import matchers from 'expect/build/matchers';
+import { currentTestInfo } from './globals';
 
 export const expect: Expect = expectLibrary as any;
 expectLibrary.setState({ expand: false });
-expectLibrary.extend({
+const customMatchers = {
   toBeChecked,
   toBeDisabled,
   toBeEditable,
@@ -66,4 +68,51 @@ expectLibrary.extend({
   toHaveURL,
   toHaveValue,
   toMatchSnapshot,
-});
+};
+
+let lastExpectSeq = 0;
+
+function wrap(matcherName: string, matcher: any) {
+  return function(this: any, ...args: any[]) {
+    const testInfo = currentTestInfo();
+    if (!testInfo)
+      return matcher.call(this, ...args);
+
+    const seq = ++lastExpectSeq;
+    testInfo._progress('expect', { phase: 'begin', seq, matcherName });
+    const endPayload: any = { phase: 'end', seq };
+    let isAsync = false;
+    try {
+      const result = matcher.call(this, ...args);
+      endPayload.pass = result.pass;
+      if (this.isNot)
+        endPayload.isNot = this.isNot;
+      if (result.pass === this.isNot && result.message)
+        endPayload.message = result.message();
+      if (result instanceof Promise) {
+        isAsync = true;
+        return result.catch(e => {
+          endPayload.error = e.stack;
+          throw e;
+        }).finally(() => {
+          testInfo._progress('expect', endPayload);
+        });
+      }
+      return result;
+    } catch (e) {
+      endPayload.error = e.stack;
+      throw e;
+    } finally {
+      if (!isAsync)
+        testInfo._progress('expect', endPayload);
+    }
+  };
+}
+
+const wrappedMatchers: any = {};
+for (const matcherName in matchers)
+  wrappedMatchers[matcherName] = wrap(matcherName, matchers[matcherName]);
+for (const matcherName in customMatchers)
+  wrappedMatchers[matcherName] = wrap(matcherName, (customMatchers as any)[matcherName]);
+
+expectLibrary.extend(wrappedMatchers);

src/test/index.ts

@@ -177,13 +177,16 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
         recordVideoDir = artifactsFolder;
       }
 
-      const combinedOptions = {
+      const combinedOptions: BrowserContextOptions = {
         recordVideo: recordVideoDir ? { dir: recordVideoDir, size: recordVideoSize } : undefined,
         ...contextOptions,
         ...options,
         ...additionalOptions,
       };
       const context = await browser.newContext(combinedOptions);
+      (context as any)._csi = {
+        onApiCall: (data: any) => (testInfo as any)._progress('pw:api', data),
+      };
       context.setDefaultTimeout(actionTimeout || 0);
       context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0);
       expectLibrary.setState({ playwrightActionTimeout: actionTimeout } as any);

src/test/ipc.ts

@@ -46,6 +46,12 @@ export type TestEndPayload = {
   attachments: { name: string, path?: string, body?: string, contentType: string }[];
 };
 
+export type ProgressPayload = {
+  testId: string;
+  name: string;
+  data: any;
+};
+
 export type TestEntry = {
   testId: string;
   retry: number;

src/test/reporters/multiplexer.ts

@@ -57,4 +57,9 @@ export class Multiplexer implements Reporter {
     for (const reporter of this._reporters)
       reporter.onError?.(error);
   }
+
+  _onTestProgress(test: TestCase, name: string, data: any) {
+    for (const reporter of this._reporters)
+      (reporter as any)._onTestProgress?.(test, name, data);
+  }
 }

src/test/types.ts

@@ -27,4 +27,5 @@ export type Annotations = { type: string, description?: string }[];
 
 export interface TestInfoImpl extends TestInfo {
   _testFinished: Promise<void>;
+  _progress: (name: string, params: any) => void;
 }

src/test/worker.ts

@@ -74,7 +74,7 @@ process.on('message', async message => {
     workerIndex = initParams.workerIndex;
     startProfiling();
     workerRunner = new WorkerRunner(initParams);
-    for (const event of ['testBegin', 'testEnd', 'done'])
+    for (const event of ['testBegin', 'testEnd', 'done', 'progress'])
       workerRunner.on(event, sendMessageToParent.bind(null, event));
     return;
   }

src/test/workerRunner.ts

@@ -267,6 +267,7 @@ export class WorkerRunner extends EventEmitter {
           deadlineRunner.setDeadline(deadline());
       },
       _testFinished: new Promise(f => testFinishedCallback = f),
+      _progress: (name, data) => this.emit('progress', { testId, name, data }),
     };
 
     // Inherit test.setTimeout() from parent suites.

tests/playwright-test/reporter.spec.ts

@@ -158,3 +158,99 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => {
     '%%end',
   ]);
 });
+
+test('should report expect progress', async ({ runInlineTest }) => {
+  const expectReporterJS = `
+    class Reporter {
+      _onTestProgress(test, name, data) {
+        if (data.frames)
+          data.frames = [];
+        console.log('%%%%', name, JSON.stringify(data));
+      }
+    }
+    module.exports = Reporter;
+  `;
+
+  const result = await runInlineTest({
+    'reporter.ts': expectReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('fail', async ({}) => {
+        expect(true).toBeTruthy();
+        expect(false).toBeTruthy();
+      });
+      test('pass', async ({}) => {
+        expect(false).not.toBeTruthy();
+      });
+      test('async', async ({ page }) => {
+        await expect(page).not.toHaveTitle('False');
+      });
+    `
+  }, { reporter: '', workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`,
+    `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":true}`,
+    `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toBeTruthy\"}`,
+    `%% expect {\"phase\":\"end\",\"seq\":2,\"pass\":false,\"message\":\"\\u001b[2mexpect(\\u001b[22m\\u001b[31mreceived\\u001b[39m\\u001b[2m).\\u001b[22mtoBeTruthy\\u001b[2m()\\u001b[22m\\n\\nReceived: \\u001b[31mfalse\\u001b[39m\"}`,
+
+    `%% expect {\"phase\":\"begin\",\"seq\":1,\"matcherName\":\"toBeTruthy\"}`,
+    `%% expect {\"phase\":\"end\",\"seq\":1,\"pass\":false,\"isNot\":true}`,
+
+    `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`,
+    `%% pw:api {\"phase\":\"end\",\"seq\":3}`,
+    `%% expect {\"phase\":\"begin\",\"seq\":2,\"matcherName\":\"toHaveTitle\"}`,
+    `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.title\",\"frames\":[]}`,
+    `%% pw:api {\"phase\":\"end\",\"seq\":4}`,
+    `%% expect {\"phase\":\"end\",\"seq\":2,\"isNot\":true}`,
+    `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"browserContext.close\",\"frames\":[]}`,
+    `%% pw:api {\"phase\":\"end\",\"seq\":5}`,
+  ]);
+});
+
+test('should report log progress', async ({ runInlineTest }) => {
+  const expectReporterJS = `
+    class Reporter {
+      _onTestProgress(test, name, data) {
+        if (data.frames)
+          data.frames = [];
+        console.log('%%%%', name, JSON.stringify(data));
+      }
+    }
+    module.exports = Reporter;
+  `;
+
+  const result = await runInlineTest({
+    'reporter.ts': expectReporterJS,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter',
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+        await page.setContent('<button></button>');
+        await page.click('button');
+      });
+    `
+  }, { reporter: '', workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    `%% pw:api {\"phase\":\"begin\",\"seq\":3,\"apiName\":\"browserContext.newPage\",\"frames\":[]}`,
+    `%% pw:api {\"phase\":\"end\",\"seq\":3}`,
+    `%% pw:api {\"phase\":\"begin\",\"seq\":4,\"apiName\":\"page.setContent\",\"frames\":[]}`,
+    `%% pw:api {\"phase\":\"end\",\"seq\":4}`,
+    `%% pw:api {\"phase\":\"begin\",\"seq\":5,\"apiName\":\"page.click\",\"frames\":[]}`,
+    `%% pw:api {\"phase\":\"end\",\"seq\":5}`,
+    `%% pw:api {\"phase\":\"begin\",\"seq\":6,\"apiName\":\"browserContext.close\",\"frames\":[]}`,
+    `%% pw:api {\"phase\":\"end\",\"seq\":6}`,
+  ]);
+});