feat(test-runner): introduce attachments (#7685)

src/test/dispatcher.ts

@@ -271,7 +271,12 @@ export class Dispatcher {
       const { test, result } = this._testById.get(params.testId)!;
       result.duration = params.duration;
       result.error = params.error;
-      result.data = params.data;
+      result.attachments = params.attachments.map(a => ({
+        name: a.name,
+        path: a.path,
+        contentType: a.contentType,
+        body: a.body ? Buffer.from(a.body, 'base64') : undefined
+      }));
       test.expectedStatus = params.expectedStatus;
       test.annotations = params.annotations;
       test.timeout = params.timeout;

src/test/expect.ts

@@ -38,7 +38,7 @@ function toMatchSnapshot(this: ReturnType<Expect['getState']>, received: Buffer
     options.threshold = projectThreshold;
 
   const withNegateComparison = this.isNot;
-  const { pass, message } = compare(
+  const { pass, message, expectedPath, actualPath, diffPath, mimeType } = compare(
       received,
       options.name,
       testInfo.snapshotPath,
@@ -47,6 +47,13 @@ function toMatchSnapshot(this: ReturnType<Expect['getState']>, received: Buffer
       withNegateComparison,
       options
   );
+  const contentType = mimeType || 'application/octet-stream';
+  if (expectedPath)
+    testInfo.attachments.push({ name: 'expected', contentType, path: expectedPath });
+  if (actualPath)
+    testInfo.attachments.push({ name: 'actual', contentType, path: actualPath });
+  if (diffPath)
+    testInfo.attachments.push({ name: 'diff', contentType, path: diffPath });
   return { pass, message: () => message };
 }
 

src/test/golden.ts

@@ -88,7 +88,7 @@ export function compare(
   updateSnapshots: UpdateSnapshots,
   withNegateComparison: boolean,
   options?: { threshold?: number }
-): { pass: boolean; message?: string; } {
+): { pass: boolean; message?: string; expectedPath?: string, actualPath?: string, diffPath?: string, mimeType?: string } {
   const snapshotFile = snapshotPath(name);
 
   if (!fs.existsSync(snapshotFile)) {
@@ -179,6 +179,10 @@ export function compare(
   return {
     pass: false,
     message: output.join('\n'),
+    expectedPath,
+    actualPath,
+    diffPath,
+    mimeType
   };
 }
 

src/test/index.ts

@@ -162,17 +162,21 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
     const preserveTrace = captureTrace && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1));
     if (preserveTrace) {
       const tracePath = testInfo.outputPath(`trace.zip`);
-      testInfo.data.playwrightTrace = tracePath;
       await context.tracing.stop({ path: tracePath });
+      testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' });
     } else if (captureTrace) {
       await context.tracing.stop();
     }
 
     const captureScreenshots = (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed));
     if (captureScreenshots) {
-      await Promise.all(allPages.map((page, index) => {
+      await Promise.all(allPages.map(async (page, index) => {
         const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${++index}.png`);
-        return page.screenshot({ timeout: 5000, path: screenshotPath }).catch(e => {});
+        try {
+          await page.screenshot({ timeout: 5000, path: screenshotPath });
+          testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' });
+        } catch {
+        }
       }));
     }
 

src/test/ipc.ts

@@ -42,7 +42,7 @@ export type TestEndPayload = {
   expectedStatus: TestStatus;
   annotations: { type: string, description?: string }[];
   timeout: number;
-  data: { [key: string]: any },
+  attachments: { name: string, path?: string, body?: string, contentType: string }[];
 };
 
 export type TestEntry = {

src/test/reporter.ts

@@ -51,7 +51,7 @@ export interface TestResult {
   duration: number;
   status?: TestStatus;
   error?: TestError;
-  data: { [key: string]: any };
+  attachments: { name: string, path?: string, body?: Buffer, contentType: string }[];
   stdout: (string | Buffer)[];
   stderr: (string | Buffer)[];
 }

src/test/reporters/json.ts

@@ -67,7 +67,7 @@ export interface JSONReportTestResult {
   stdout: JSONReportSTDIOEntry[],
   stderr: JSONReportSTDIOEntry[],
   retry: number;
-  data: { [key: string]: any },
+  attachments: { name: string, path?: string, body?: string, contentType: string }[];
 }
 export type JSONReportSTDIOEntry = { text: string } | { buffer: string };
 
@@ -217,7 +217,12 @@ class JSONReporter implements Reporter {
       stdout: result.stdout.map(s => stdioEntry(s)),
       stderr: result.stderr.map(s => stdioEntry(s)),
       retry: result.retry,
-      data: result.data,
+      attachments: result.attachments.map(a => ({
+        name: a.name,
+        contentType: a.contentType,
+        path: a.path,
+        body: a.body?.toString('base64')
+      })),
     };
   }
 }

src/test/test.ts

@@ -187,7 +187,7 @@ export class Test extends Base implements reporterTypes.Test {
       duration: 0,
       stdout: [],
       stderr: [],
-      data: {},
+      attachments: [],
     };
     this.results.push(result);
     return result;

src/test/workerRunner.ts

@@ -226,7 +226,7 @@ export class WorkerRunner extends EventEmitter {
       retry: entry.retry,
       expectedStatus: 'passed',
       annotations: [],
-      data: {},
+      attachments: [],
       duration: 0,
       status: 'passed',
       stdout: [],
@@ -477,7 +477,12 @@ function buildTestEndPayload(testId: string, testInfo: TestInfo): TestEndPayload
     expectedStatus: testInfo.expectedStatus,
     annotations: testInfo.annotations,
     timeout: testInfo.timeout,
-    data: testInfo.data,
+    attachments: testInfo.attachments.map(a => ({
+      name: a.name,
+      contentType: a.contentType,
+      path: a.path,
+      body: a.body?.toString('base64')
+    }))
   };
 }
 

tests/playwright-test/access-data.spec.ts

@@ -83,13 +83,13 @@ test('should report projectName in result', async ({ runInlineTest }) => {
   expect(exitCode).toBe(0);
 });
 
-test('should access testInfo.data in fixture', async ({ runInlineTest }) => {
+test('should access testInfo.attachments in fixture', async ({ runInlineTest }) => {
   const { exitCode, report } = await runInlineTest({
     'test-data-visible-in-env.spec.ts': `
       const test = pwt.test.extend({
         foo: async ({}, run, testInfo) => {
           await run();
-          testInfo.data.foo = 'bar';
+          testInfo.attachments.push({ name: 'foo', body: Buffer.from([1, 2, 3]), contentType: 'application/octet-stream' });
         },
       });
       test('ensure fixture can set data', async ({ foo }) => {
@@ -98,5 +98,5 @@ test('should access testInfo.data in fixture', async ({ runInlineTest }) => {
   });
   expect(exitCode).toBe(0);
   const test = report.suites[0].specs[0].tests[0];
-  expect(test.results[0].data).toEqual({ foo: 'bar' });
+  expect(test.results[0].attachments[0]).toEqual({ name: 'foo', body: 'AQID', contentType: 'application/octet-stream' });
 });

types/test.d.ts

@@ -399,9 +399,9 @@ export interface TestInfo extends WorkerInfo {
   annotations: { type: string, description?: string }[];
 
   /**
-   * Arbitrary data that test fixtures can provide for the test report.
+   * File attachments for this test.
    */
-  data: { [key: string]: any };
+  attachments: { name: string, path?: string, body?: Buffer, contentType: string }[];
 
   /**
    * When tests are run multiple times, each run gets a unique `repeatEachIndex`.