From bde764085c1c80696ec5e823f7a9898e1abddc9b Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 16 Jul 2021 13:48:37 -0700 Subject: [PATCH] feat(test-runner): introduce attachments (#7685) --- src/test/dispatcher.ts | 7 ++++++- src/test/expect.ts | 9 ++++++++- src/test/golden.ts | 6 +++++- src/test/index.ts | 10 +++++++--- src/test/ipc.ts | 2 +- src/test/reporter.ts | 2 +- src/test/reporters/json.ts | 9 +++++++-- src/test/test.ts | 2 +- src/test/workerRunner.ts | 9 +++++++-- tests/playwright-test/access-data.spec.ts | 6 +++--- types/test.d.ts | 4 ++-- 11 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/test/dispatcher.ts b/src/test/dispatcher.ts index 9c1f9fe4c3..10d02964e8 100644 --- a/src/test/dispatcher.ts +++ b/src/test/dispatcher.ts @@ -271,7 +271,12 @@ export class Dispatcher { const { test, result } = this._testById.get(params.testId)!; result.duration = params.duration; result.error = params.error; - result.data = params.data; + result.attachments = params.attachments.map(a => ({ + name: a.name, + path: a.path, + contentType: a.contentType, + body: a.body ? Buffer.from(a.body, 'base64') : undefined + })); test.expectedStatus = params.expectedStatus; test.annotations = params.annotations; test.timeout = params.timeout; diff --git a/src/test/expect.ts b/src/test/expect.ts index f73c9f4b9a..dd15c1316b 100644 --- a/src/test/expect.ts +++ b/src/test/expect.ts @@ -38,7 +38,7 @@ function toMatchSnapshot(this: ReturnType, received: Buffer options.threshold = projectThreshold; const withNegateComparison = this.isNot; - const { pass, message } = compare( + const { pass, message, expectedPath, actualPath, diffPath, mimeType } = compare( received, options.name, testInfo.snapshotPath, @@ -47,6 +47,13 @@ function toMatchSnapshot(this: ReturnType, received: Buffer withNegateComparison, options ); + const contentType = mimeType || 'application/octet-stream'; + if (expectedPath) + testInfo.attachments.push({ name: 'expected', contentType, path: expectedPath }); + if (actualPath) + testInfo.attachments.push({ name: 'actual', contentType, path: actualPath }); + if (diffPath) + testInfo.attachments.push({ name: 'diff', contentType, path: diffPath }); return { pass, message: () => message }; } diff --git a/src/test/golden.ts b/src/test/golden.ts index 460a29aa75..ea29ea5bfa 100644 --- a/src/test/golden.ts +++ b/src/test/golden.ts @@ -88,7 +88,7 @@ export function compare( updateSnapshots: UpdateSnapshots, withNegateComparison: boolean, options?: { threshold?: number } -): { pass: boolean; message?: string; } { +): { pass: boolean; message?: string; expectedPath?: string, actualPath?: string, diffPath?: string, mimeType?: string } { const snapshotFile = snapshotPath(name); if (!fs.existsSync(snapshotFile)) { @@ -179,6 +179,10 @@ export function compare( return { pass: false, message: output.join('\n'), + expectedPath, + actualPath, + diffPath, + mimeType }; } diff --git a/src/test/index.ts b/src/test/index.ts index f7b55a4567..7b1a79963c 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -162,17 +162,21 @@ export const test = _baseTest.extend { + await Promise.all(allPages.map(async (page, index) => { const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${++index}.png`); - return page.screenshot({ timeout: 5000, path: screenshotPath }).catch(e => {}); + try { + await page.screenshot({ timeout: 5000, path: screenshotPath }); + testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' }); + } catch { + } })); } diff --git a/src/test/ipc.ts b/src/test/ipc.ts index 935c0672e8..e02c3f1ed9 100644 --- a/src/test/ipc.ts +++ b/src/test/ipc.ts @@ -42,7 +42,7 @@ export type TestEndPayload = { expectedStatus: TestStatus; annotations: { type: string, description?: string }[]; timeout: number; - data: { [key: string]: any }, + attachments: { name: string, path?: string, body?: string, contentType: string }[]; }; export type TestEntry = { diff --git a/src/test/reporter.ts b/src/test/reporter.ts index ff755db44b..3246b71eb0 100644 --- a/src/test/reporter.ts +++ b/src/test/reporter.ts @@ -51,7 +51,7 @@ export interface TestResult { duration: number; status?: TestStatus; error?: TestError; - data: { [key: string]: any }; + attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; stdout: (string | Buffer)[]; stderr: (string | Buffer)[]; } diff --git a/src/test/reporters/json.ts b/src/test/reporters/json.ts index 25557afcb2..7240f34b60 100644 --- a/src/test/reporters/json.ts +++ b/src/test/reporters/json.ts @@ -67,7 +67,7 @@ export interface JSONReportTestResult { stdout: JSONReportSTDIOEntry[], stderr: JSONReportSTDIOEntry[], retry: number; - data: { [key: string]: any }, + attachments: { name: string, path?: string, body?: string, contentType: string }[]; } export type JSONReportSTDIOEntry = { text: string } | { buffer: string }; @@ -217,7 +217,12 @@ class JSONReporter implements Reporter { stdout: result.stdout.map(s => stdioEntry(s)), stderr: result.stderr.map(s => stdioEntry(s)), retry: result.retry, - data: result.data, + attachments: result.attachments.map(a => ({ + name: a.name, + contentType: a.contentType, + path: a.path, + body: a.body?.toString('base64') + })), }; } } diff --git a/src/test/test.ts b/src/test/test.ts index 4cc802f7d3..987906ea8b 100644 --- a/src/test/test.ts +++ b/src/test/test.ts @@ -187,7 +187,7 @@ export class Test extends Base implements reporterTypes.Test { duration: 0, stdout: [], stderr: [], - data: {}, + attachments: [], }; this.results.push(result); return result; diff --git a/src/test/workerRunner.ts b/src/test/workerRunner.ts index 8a354f7fce..7693f81f11 100644 --- a/src/test/workerRunner.ts +++ b/src/test/workerRunner.ts @@ -226,7 +226,7 @@ export class WorkerRunner extends EventEmitter { retry: entry.retry, expectedStatus: 'passed', annotations: [], - data: {}, + attachments: [], duration: 0, status: 'passed', stdout: [], @@ -477,7 +477,12 @@ function buildTestEndPayload(testId: string, testInfo: TestInfo): TestEndPayload expectedStatus: testInfo.expectedStatus, annotations: testInfo.annotations, timeout: testInfo.timeout, - data: testInfo.data, + attachments: testInfo.attachments.map(a => ({ + name: a.name, + contentType: a.contentType, + path: a.path, + body: a.body?.toString('base64') + })) }; } diff --git a/tests/playwright-test/access-data.spec.ts b/tests/playwright-test/access-data.spec.ts index f9b7b6572b..b494572364 100644 --- a/tests/playwright-test/access-data.spec.ts +++ b/tests/playwright-test/access-data.spec.ts @@ -83,13 +83,13 @@ test('should report projectName in result', async ({ runInlineTest }) => { expect(exitCode).toBe(0); }); -test('should access testInfo.data in fixture', async ({ runInlineTest }) => { +test('should access testInfo.attachments in fixture', async ({ runInlineTest }) => { const { exitCode, report } = await runInlineTest({ 'test-data-visible-in-env.spec.ts': ` const test = pwt.test.extend({ foo: async ({}, run, testInfo) => { await run(); - testInfo.data.foo = 'bar'; + testInfo.attachments.push({ name: 'foo', body: Buffer.from([1, 2, 3]), contentType: 'application/octet-stream' }); }, }); test('ensure fixture can set data', async ({ foo }) => { @@ -98,5 +98,5 @@ test('should access testInfo.data in fixture', async ({ runInlineTest }) => { }); expect(exitCode).toBe(0); const test = report.suites[0].specs[0].tests[0]; - expect(test.results[0].data).toEqual({ foo: 'bar' }); + expect(test.results[0].attachments[0]).toEqual({ name: 'foo', body: 'AQID', contentType: 'application/octet-stream' }); }); diff --git a/types/test.d.ts b/types/test.d.ts index c60b345f46..960b117d74 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -399,9 +399,9 @@ export interface TestInfo extends WorkerInfo { annotations: { type: string, description?: string }[]; /** - * Arbitrary data that test fixtures can provide for the test report. + * File attachments for this test. */ - data: { [key: string]: any }; + attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; /** * When tests are run multiple times, each run gets a unique `repeatEachIndex`.