feat(report): render attachment as a part of failure (#8903)

src/test/reporters/base.ts

@@ -153,17 +153,41 @@ export class BaseReporter implements Reporter  {
 }
 
 export function formatFailure(config: FullConfig, test: TestCase, index?: number, stdio?: boolean): string {
-  const tokens: string[] = [];
-  tokens.push(formatTestHeader(config, test, '  ', index));
+  const lines: string[] = [];
+  lines.push(formatTestHeader(config, test, '  ', index));
   for (const result of test.results) {
     const resultTokens = formatResultFailure(test, result, '    ');
     if (!resultTokens.length)
       continue;
     if (result.retry) {
-      tokens.push('');
-      tokens.push(colors.gray(pad(`    Retry #${result.retry}`, '-')));
+      lines.push('');
+      lines.push(colors.gray(pad(`    Retry #${result.retry}`, '-')));
+    }
+    lines.push(...resultTokens);
+    for (let i = 0; i < result.attachments.length; ++i) {
+      const attachment = result.attachments[i];
+      lines.push('');
+      lines.push(colors.cyan(pad(`    attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`, '-')));
+      if (attachment.path) {
+        const relativePath = path.relative(process.cwd(), attachment.path);
+        lines.push(colors.cyan(`    ${relativePath}`));
+        // Make this extensible
+        if (attachment.name === 'trace') {
+          lines.push(colors.cyan(`    Usage:`));
+          lines.push('');
+          lines.push(colors.cyan(`        npx playwright show-trace ${relativePath}`));
+          lines.push('');
+        }
+      } else {
+        if (attachment.contentType.startsWith('text/')) {
+          let text = attachment.body!.toString();
+          if (text.length > 300)
+            text = text.slice(0, 300) + '...';
+          lines.push(colors.cyan(`    ${text}`));
+        }
+      }
+      lines.push(colors.cyan(pad('   ', '-')));
     }
-    tokens.push(...resultTokens);
     const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[];
     if (stdio && output.length) {
       const outputText = output.map(({ chunk, type }) => {
@@ -172,12 +196,12 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number
           return colors.red(stripAnsiEscapes(text));
         return text;
       }).join('');
-      tokens.push('');
-      tokens.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-'));
+      lines.push('');
+      lines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-'));
     }
   }
-  tokens.push('');
-  return tokens.join('\n');
+  lines.push('');
+  return lines.join('\n');
 }
 
 export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string): string[] {

tests/config/browserTest.ts

@@ -151,8 +151,11 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
     });
     await Promise.all(contexts.map(async context => {
       const videos = context.pages().map(p => p.video()).filter(Boolean);
-      if (!(context as any)._closed && trace)
-        await context.tracing.stop({ path: testInfo.outputPath('trace.zip') });
+      if (!(context as any)._closed && trace) {
+        const tracePath = testInfo.outputPath('trace.zip');
+        await context.tracing.stop({ path: tracePath });
+        testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' });
+      }
       await context.close();
       for (const v of videos) {
         const videoPath = await v.path().catch(() => null);

tests/playwright-test/attachment-reporter.spec.ts

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+
+test('render text attachment', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('one', async ({}, testInfo) => {
+        testInfo.attachments.push({
+          name: 'attachment',
+          body: Buffer.from('Hello world'),
+          contentType: 'text/plain'
+        });
+        expect(1).toBe(0);
+      });
+    `,
+  }, { reporter: 'line' });
+  const text = stripAscii(result.output);
+  expect(text).toContain('    attachment #1: attachment (text/plain) ---------------------------------------------------------');
+  expect(text).toContain('    Hello world');
+  expect(text).toContain('    ------------------------------------------------------------------------------------------------');
+  expect(result.exitCode).toBe(1);
+});
+
+test('render screenshot attachment', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('one', async ({}, testInfo) => {
+        testInfo.attachments.push({
+          name: 'screenshot',
+          path: testInfo.outputPath('some/path.png'),
+          contentType: 'image/png'
+        });
+        expect(1).toBe(0);
+      });
+    `,
+  }, { reporter: 'line' });
+  const text = stripAscii(result.output).replace(/\\/g, '/');
+  expect(text).toContain('    attachment #1: screenshot (image/png) ----------------------------------------------------------');
+  expect(text).toContain('    test-results/a-one/some/path.png');
+  expect(text).toContain('    ------------------------------------------------------------------------------------------------');
+  expect(result.exitCode).toBe(1);
+});
+
+test('render trace attachment', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('one', async ({}, testInfo) => {
+        testInfo.attachments.push({
+          name: 'trace',
+          path: testInfo.outputPath('trace.zip'),
+          contentType: 'application/zip'
+        });
+        expect(1).toBe(0);
+      });
+    `,
+  }, { reporter: 'line' });
+  const text = stripAscii(result.output).replace(/\\/g, '/');
+  expect(text).toContain('    attachment #1: trace (application/zip) ---------------------------------------------------------');
+  expect(text).toContain('    test-results/a-one/trace.zip');
+  expect(text).toContain('npx playwright show-trace test-results/a-one/trace.zip');
+  expect(text).toContain('    ------------------------------------------------------------------------------------------------');
+  expect(result.exitCode).toBe(1);
+});