feat(test runner): export testInfo.data (#7525)

This is a key-value storage for any information that goes into the report.
Also export JSONReport types.

src/test/dispatcher.ts

@@ -275,6 +275,7 @@ export class Dispatcher {
       const { test, result } = this._testById.get(params.testId)!;
       result.duration = params.duration;
       result.error = params.error;
+      result.data = params.data;
       test.expectedStatus = params.expectedStatus;
       test.annotations = params.annotations;
       test.timeout = params.timeout;

src/test/index.ts

@@ -162,6 +162,7 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
     const preserveTrace = captureTrace && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1));
     if (preserveTrace) {
       const tracePath = testInfo.outputPath(`trace.zip`);
+      testInfo.data.playwrightTrace = tracePath;
       await context.tracing.stop({ path: tracePath });
     } else if (captureTrace) {
       await context.tracing.stop();

src/test/ipc.ts

@@ -42,6 +42,7 @@ export type TestEndPayload = {
   expectedStatus: TestStatus;
   annotations: { type: string, description?: string }[];
   timeout: number;
+  data: { [key: string]: any },
 };
 
 export type TestEntry = {

src/test/reporter.ts

@@ -57,6 +57,7 @@ export interface TestResult {
   duration: number;
   status?: TestStatus;
   error?: TestError;
+  data: { [key: string]: any };
   stdout: (string | Buffer)[];
   stderr: (string | Buffer)[];
 }

src/test/reporters/json.ts

@@ -17,18 +17,60 @@
 import fs from 'fs';
 import path from 'path';
 import EmptyReporter from './empty';
-import { FullConfig, Test, Suite, Spec, TestResult, TestError, FullResult } from '../reporter';
+import { FullConfig, Test, Suite, Spec, TestResult, TestError, FullResult, TestStatus } from '../reporter';
 
-interface SerializedSuite {
+export interface JSONReport {
+  config: Omit<FullConfig, 'projects'> & {
+    projects: {
+      outputDir: string,
+      repeatEach: number,
+      retries: number,
+      metadata: any,
+      name: string,
+      testDir: string,
+      testIgnore: string[],
+      testMatch: string[],
+      timeout: number,
+    }[],
+  };
+  suites: JSONReportSuite[];
+  errors: TestError[];
+}
+export interface JSONReportSuite {
   title: string;
   file: string;
   column: number;
   line: number;
-  specs: ReturnType<JSONReporter['_serializeTestSpec']>[];
-  suites?: SerializedSuite[];
+  specs: JSONReportSpec[];
+  suites?: JSONReportSuite[];
 }
-
-export type ReportFormat = ReturnType<JSONReporter['_serializeReport']>;
+export interface JSONReportSpec {
+  title: string;
+  ok: boolean;
+  tests: JSONReportTest[];
+  file: string;
+  line: number;
+  column: number;
+}
+export interface JSONReportTest {
+  timeout: number;
+  annotations: { type: string, description?: string }[],
+  expectedStatus: TestStatus;
+  projectName: string;
+  results: JSONReportTestResult[];
+  status: 'skipped' | 'expected' | 'unexpected' | 'flaky';
+}
+export interface JSONReportTestResult {
+  workerIndex: number;
+  status: TestStatus | undefined;
+  duration: number;
+  error: TestError | undefined;
+  stdout: JSONReportSTDIOEntry[],
+  stderr: JSONReportSTDIOEntry[],
+  retry: number;
+  data: { [key: string]: any },
+}
+export type JSONReportSTDIOEntry = { text: string } | { buffer: string };
 
 function toPosixPath(aPath: string): string {
   return aPath.split(path.sep).join(path.posix.sep);
@@ -58,7 +100,7 @@ class JSONReporter extends EmptyReporter {
     outputReport(this._serializeReport(), this._outputFile);
   }
 
-  private _serializeReport() {
+  private _serializeReport(): JSONReport {
     return {
       config: {
         ...this.config,
@@ -77,15 +119,15 @@ class JSONReporter extends EmptyReporter {
           };
         })
       },
-      suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s),
+      suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[],
       errors: this._errors
     };
   }
 
-  private _serializeSuite(suite: Suite): null | SerializedSuite {
+  private _serializeSuite(suite: Suite): null | JSONReportSuite {
     if (!suite.findSpec(test => true))
       return null;
-    const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as SerializedSuite[];
+    const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[];
     return {
       title: suite.title,
       file: toPosixPath(path.relative(this.config.rootDir, suite.file)),
@@ -96,7 +138,7 @@ class JSONReporter extends EmptyReporter {
     };
   }
 
-  private _serializeTestSpec(spec: Spec) {
+  private _serializeTestSpec(spec: Spec): JSONReportSpec {
     return {
       title: spec.title,
       ok: spec.ok(),
@@ -107,17 +149,18 @@ class JSONReporter extends EmptyReporter {
     };
   }
 
-  private _serializeTest(test: Test) {
+  private _serializeTest(test: Test): JSONReportTest {
     return {
       timeout: test.timeout,
       annotations: test.annotations,
       expectedStatus: test.expectedStatus,
       projectName: test.projectName,
       results: test.results.map(r => this._serializeTestResult(r)),
+      status: test.status(),
     };
   }
 
-  private _serializeTestResult(result: TestResult) {
+  private _serializeTestResult(result: TestResult): JSONReportTestResult {
     return {
       workerIndex: result.workerIndex,
       status: result.status,
@@ -126,11 +169,12 @@ class JSONReporter extends EmptyReporter {
       stdout: result.stdout.map(s => stdioEntry(s)),
       stderr: result.stderr.map(s => stdioEntry(s)),
       retry: result.retry,
+      data: result.data,
     };
   }
 }
 
-function outputReport(report: ReportFormat, outputFile: string | undefined) {
+function outputReport(report: JSONReport, outputFile: string | undefined) {
   const reportString = JSON.stringify(report, undefined, 2);
   outputFile = outputFile || process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`];
   if (outputFile) {

src/test/test.ts

@@ -227,6 +227,7 @@ export class Test implements reporterTypes.Test {
       duration: 0,
       stdout: [],
       stderr: [],
+      data: {},
     };
     this.results.push(result);
     return result;

src/test/workerRunner.ts

@@ -225,6 +225,7 @@ export class WorkerRunner extends EventEmitter {
       retry: entry.retry,
       expectedStatus: 'passed',
       annotations: [],
+      data: {},
       duration: 0,
       status: 'passed',
       stdout: [],
@@ -484,6 +485,7 @@ function buildTestEndPayload(testId: string, testInfo: TestInfo): TestEndPayload
     expectedStatus: testInfo.expectedStatus,
     annotations: testInfo.annotations,
     timeout: testInfo.timeout,
+    data: testInfo.data,
   };
 }
 

tests/playwright-test/access-data.spec.ts

@@ -82,3 +82,21 @@ test('should report projectName in result', async ({ runInlineTest }) => {
   expect(report.suites[0].specs[0].tests[1].projectName).toBe('');
   expect(exitCode).toBe(0);
 });
+
+test('should access testInfo.data in fixture', async ({ runInlineTest }) => {
+  const { exitCode, report } = await runInlineTest({
+    'test-data-visible-in-env.spec.ts': `
+      const test = pwt.test.extend({
+        foo: async ({}, run, testInfo) => {
+          await run();
+          testInfo.data.foo = 'bar';
+        },
+      });
+      test('ensure fixture can set data', async ({ foo }) => {
+      });
+    `
+  });
+  expect(exitCode).toBe(0);
+  const test = report.suites[0].specs[0].tests[0];
+  expect(test.results[0].data).toEqual({ foo: 'bar' });
+});

tests/playwright-test/playwright-test-fixtures.ts

@@ -19,7 +19,7 @@ import { spawn } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
-import type { ReportFormat } from '../../src/test/reporters/json';
+import type { JSONReport, JSONReportSuite } from '../../src/test/reporters/json';
 import rimraf from 'rimraf';
 import { promisify } from 'util';
 import * as url from 'url';
@@ -33,7 +33,7 @@ type RunResult = {
   failed: number,
   flaky: number,
   skipped: number,
-  report: ReportFormat,
+  report: JSONReport,
   results: any[],
 };
 
@@ -191,7 +191,7 @@ async function runPlaywrightTest(baseDir: string, params: any, env: Env, options
   }
 
   const results = [];
-  function visitSuites(suites?: ReportFormat['suites']) {
+  function visitSuites(suites?: JSONReportSuite[]) {
     if (!suites)
       return;
     for (const suite of suites) {

types/test.d.ts

@@ -124,7 +124,7 @@ export type WebServerConfig = {
    */
   command: string,
   /**
-   * The port that your server is expected to appear on. If not specified, it does get automatically collected via the 
+   * The port that your server is expected to appear on. If not specified, it does get automatically collected via the
    * command output when a localhost URL gets printed.
    */
   port?: number,
@@ -394,6 +394,11 @@ export interface TestInfo extends WorkerInfo {
    */
   annotations: { type: string, description?: string }[];
 
+  /**
+   * Arbitrary data that test fixtures can provide for the test report.
+   */
+  data: { [key: string]: any };
+
   /**
    * When tests are run multiple times, each run gets a unique `repeatEachIndex`.
    */