From 854f321532c6ce3e5bf38da8d56c7e6b882af6aa Mon Sep 17 00:00:00 2001 From: Ross Wollman Date: Tue, 23 Nov 2021 09:30:53 -0800 Subject: [PATCH] feat(api): add explicit async testInfo.attach (#10121) feat(api): add explicit async testInfo.attach We add an explicit async API for attaching file paths (and Buffers) to tests that can be awaited to help users ensure they are attaching files that actually exist at both the time of the invocation and later when reporters (like the HTML Reporter) run and package up test artifacts. This is intended to help surface attachment issues as soon as possible so you aren't silently left with a missing attachment minutes/days/months later when you go to debug a suddenly breaking test expecting an attachment to be there. NB: The current implemntation incurs an extra file copy compared to manipulating the raw attachments array. If users encounter performance issues because of this, we can consider an option parameter that uses rename under the hood instead of copy. However, that would need to be used with care if the file were to be accessed later in the test. --- docs/src/test-api/class-testinfo.md | 72 ++++++- packages/playwright-test/src/workerRunner.ts | 36 ++++ packages/playwright-test/types/test.d.ts | 47 ++++- .../reporter-attachment.spec.ts | 32 ++++ tests/playwright-test/reporter-raw.spec.ts | 175 ++++++++++++++++++ utils/generate_types/overrides-test.d.ts | 2 + 6 files changed, 357 insertions(+), 7 deletions(-) diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index 551778e1a5..349ee47d61 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -38,7 +38,14 @@ Learn more about [test annotations](./test-annotations.md). - `path` <[void]|[string]> Optional path on the filesystem to the attached file. - `body` <[void]|[Buffer]> Optional attachment body used instead of a file. -The list of files or buffers attached to the current test. Some reporters show test attachments. For example, you can attach a screenshot to the test. +The list of files or buffers attached to the current test. Some reporters show test attachments. + +To safely add a file from disk as an attachment, please use [`method: TestInfo.attach#1`] instead of directly pushing onto this array. For inline attachments, use [`method: TestInfo.attach#1`]. + +## method: TestInfo.attach#1 +Attach a file from disk to the current test. Some reporters show test attachments. The [`option: name`] and [`option: contentType`] will be inferred by default from the [`param: path`], but you can optionally override either of these. + +For example, you can attach a screenshot to the test: ```js js-flavor=js const { test, expect } = require('@playwright/test'); @@ -49,7 +56,11 @@ test('basic test', async ({ page }, testInfo) => { // Capture a screenshot and attach it. const path = testInfo.outputPath('screenshot.png'); await page.screenshot({ path }); - testInfo.attachments.push({ name: 'screenshot', path, contentType: 'image/png' }); + await testInfo.attach(path); + // Optionally override the name. + await testInfo.attach(path, { name: 'example.png' }); + // Optionally override the contentType. + await testInfo.attach(path, { name: 'example.custom-file', contentType: 'x-custom-content-type' }); }); ``` @@ -62,10 +73,65 @@ test('basic test', async ({ page }, testInfo) => { // Capture a screenshot and attach it. const path = testInfo.outputPath('screenshot.png'); await page.screenshot({ path }); - testInfo.attachments.push({ name: 'screenshot', path, contentType: 'image/png' }); + await testInfo.attach(path); + // Optionally override the name. + await testInfo.attach(path, { name: 'example.png' }); + // Optionally override the contentType. + await testInfo.attach(path, { name: 'example.custom-file', contentType: 'x-custom-content-type' }); }); ``` +Or you can attach files returned by your APIs: + +```js js-flavor=js +const { test, expect } = require('@playwright/test'); + +test('basic test', async ({}, testInfo) => { + const { download } = require('./my-custom-helpers'); + const tmpPath = await download('a'); + await testInfo.attach(tmpPath, { name: 'example.json' }); +}); +``` + +```js js-flavor=ts +import { test, expect } from '@playwright/test'; + +test('basic test', async ({}, testInfo) => { + const { download } = require('./my-custom-helpers'); + const tmpPath = await download('a'); + await testInfo.attach(tmpPath, { name: 'example.json' }); +}); +``` + +:::note +[`method: TestInfo.attach#1`] automatically takes care of copying attachments to a +location that is accessible to reporters, even if you were to delete the attachment +after awaiting the attach call. +::: + +### param: TestInfo.attach#1.path +- `path` <[string]> Path on the filesystem to the attached file. + +### option: TestInfo.attach#1.name +- `name` <[void]|[string]> Optional attachment name. If omitted, this will be inferred from [`param: path`]. + +### option: TestInfo.attach#1.contentType +- `contentType` <[void]|[string]> Optional content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. If omitted, this falls back to an inferred type based on the [`param: name`] (if set) or [`param: path`]'s extension; it will be set to `application/octet-stream` if the type cannot be inferred from the file extension. + + +## method: TestInfo.attach#2 + +Attach data to the current test, either a `string` or a `Buffer`. Some reporters show test attachments. + +### param: TestInfo.attach#2.body +- `body` <[string]|[Buffer]> Attachment body. + +### param: TestInfo.attach#2.name +- `name` <[string]> Attachment name. + +### option: TestInfo.attach#2.contentType +- `contentType` <[void]|[string]> Optional content type of this attachment to properly present in the report, for example `'application/json'` or `'application/xml'`. If omitted, this falls back to an inferred type based on the [`param: name`]'s extension; if the type cannot be inferred from the name's extension, it will be set to `text/plain` (if [`param: body`] is a `string`) or `application/octet-stream` (if [`param: body`] is a `Buffer`). + ## property: TestInfo.column - type: <[int]> diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index bfa2ac00de..fe93cf4a1d 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -17,6 +17,7 @@ import fs from 'fs'; import path from 'path'; import rimraf from 'rimraf'; +import * as mime from 'mime'; import util from 'util'; import colors from 'colors/safe'; import { EventEmitter } from 'events'; @@ -29,6 +30,7 @@ import { Annotations, TestError, TestInfo, TestInfoImpl, TestStepInternal, Worke import { ProjectImpl } from './project'; import { FixtureRunner } from './fixtures'; import { DeadlineRunner, raceAgainstDeadline } from 'playwright-core/lib/utils/async'; +import { calculateFileSha1 } from 'playwright-core/lib/utils/utils'; const removeFolderAsync = util.promisify(rimraf); @@ -262,6 +264,40 @@ export class WorkerRunner extends EventEmitter { expectedStatus: test.expectedStatus, annotations: [], attachments: [], + attach: async (...args) => { + const [ pathOrBody, nameOrFileOptions, inlineOptions ] = args as [string | Buffer, string | { contentType?: string, name?: string} | undefined, { contentType?: string } | undefined]; + let attachment: { name: string, contentType: string, body?: Buffer, path?: string } | undefined; + if (typeof nameOrFileOptions === 'string') { // inline attachment + const body = pathOrBody; + const name = nameOrFileOptions; + + attachment = { + name, + contentType: inlineOptions?.contentType ?? (mime.getType(name) || (typeof body === 'string' ? 'text/plain' : 'application/octet-stream')), + body: typeof body === 'string' ? Buffer.from(body) : body, + }; + } else { // path based attachment + const options = nameOrFileOptions; + const thePath = pathOrBody as string; + const name = options?.name ?? path.basename(thePath); + attachment = { + name, + path: thePath, + contentType: options?.contentType ?? (mime.getType(name) || 'application/octet-stream') + }; + } + + const tmpAttachment = { ...attachment }; + if (attachment.path) { + const hash = await calculateFileSha1(attachment.path); + const dest = testInfo.outputPath('attachments', hash + path.extname(attachment.path)); + await fs.promises.mkdir(path.dirname(dest), { recursive: true }); + await fs.promises.copyFile(attachment.path, dest); + tmpAttachment.path = dest; + } + + testInfo.attachments.push(tmpAttachment); + }, duration: 0, status: 'passed', stdout: [], diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index ccbaa9febb..4661971bcc 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -1301,8 +1301,19 @@ export interface TestInfo { */ annotations: { type: string, description?: string }[]; /** - * The list of files or buffers attached to the current test. Some reporters show test attachments. For example, you can - * attach a screenshot to the test. + * The list of files or buffers attached to the current test. Some reporters show test attachments. + * + * To safely add a file from disk as an attachment, please use + * [testInfo.attach(path[, options])](https://playwright.dev/docs/api/class-testinfo#test-info-attach-1) instead of + * directly pushing onto this array. For inline attachments, use + * [testInfo.attach(path[, options])](https://playwright.dev/docs/api/class-testinfo#test-info-attach-1). + */ + attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; + /** + * Attach a file from disk to the current test. Some reporters show test attachments. The `name` and `contentType` will be + * inferred by default from the `path`, but you can optionally override either of these. + * + * For example, you can attach a screenshot to the test: * * ```ts * import { test, expect } from '@playwright/test'; @@ -1313,12 +1324,40 @@ export interface TestInfo { * // Capture a screenshot and attach it. * const path = testInfo.outputPath('screenshot.png'); * await page.screenshot({ path }); - * testInfo.attachments.push({ name: 'screenshot', path, contentType: 'image/png' }); + * await testInfo.attach(path); + * // Optionally override the name. + * await testInfo.attach(path, { name: 'example.png' }); + * // Optionally override the contentType. + * await testInfo.attach(path, { name: 'example.custom-file', contentType: 'x-custom-content-type' }); * }); * ``` * + * Or you can attach files returned by your APIs: + * + * ```ts + * import { test, expect } from '@playwright/test'; + * + * test('basic test', async ({}, testInfo) => { + * const { download } = require('./my-custom-helpers'); + * const tmpPath = await download('a'); + * await testInfo.attach(tmpPath, { name: 'example.json' }); + * }); + * ``` + * + * > NOTE: [testInfo.attach(path[, options])](https://playwright.dev/docs/api/class-testinfo#test-info-attach-1) + * automatically takes care of copying attachments to a location that is accessible to reporters, even if you were to + * delete the attachment after awaiting the attach call. + * @param path + * @param options */ - attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; + attach(path: string, options?: { contentType?: string, name?: string}): Promise; + /** + * Attach data to the current test, either a `string` or a `Buffer`. Some reporters show test attachments. + * @param body + * @param name + * @param options + */ + attach(body: string | Buffer, name: string, options?: { contentType?: string }): Promise; /** * Specifies a unique repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to * the [command line](https://playwright.dev/docs/test-cli). diff --git a/tests/playwright-test/reporter-attachment.spec.ts b/tests/playwright-test/reporter-attachment.spec.ts index 9e50a3c891..6b23eacfea 100644 --- a/tests/playwright-test/reporter-attachment.spec.ts +++ b/tests/playwright-test/reporter-attachment.spec.ts @@ -79,3 +79,35 @@ test('render trace attachment', async ({ runInlineTest }) => { expect(text).toContain(' ------------------------------------------------------------------------------------------------'); expect(result.exitCode).toBe(1); }); + + +test(`testInfo.attach throws an error when attaching a non-existent attachment`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('all options specified', async ({}, testInfo) => { + await testInfo.attach('non-existent-path-all-options', { contentType: 'text/plain', name: 'foo.txt'}); + }); + + test('no options specified', async ({}, testInfo) => { + await testInfo.attach('non-existent-path-no-options'); + }); + + test('partial options - contentType', async ({}, testInfo) => { + await testInfo.attach('non-existent-path-partial-options-content-type', { contentType: 'text/plain'}); + }); + + test('partial options - name', async ({}, testInfo) => { + await testInfo.attach('non-existent-path-partial-options-name', { name: 'foo.txt'}); + }); + `, + }, { reporter: 'line', workers: 1 }); + const text = stripAscii(result.output).replace(/\\/g, '/'); + expect(text).toMatch(/Error: ENOENT: no such file or directory, open '.*non-existent-path-all-options.*'/); + expect(text).toMatch(/Error: ENOENT: no such file or directory, open '.*non-existent-path-no-options.*'/); + expect(text).toMatch(/Error: ENOENT: no such file or directory, open '.*non-existent-path-partial-options-content-type.*'/); + expect(text).toMatch(/Error: ENOENT: no such file or directory, open '.*non-existent-path-partial-options-name.*'/); + expect(result.passed).toBe(0); + expect(result.failed).toBe(4); + expect(result.exitCode).toBe(1); +}); diff --git a/tests/playwright-test/reporter-raw.spec.ts b/tests/playwright-test/reporter-raw.spec.ts index e6a492a48d..546c8dfb91 100644 --- a/tests/playwright-test/reporter-raw.spec.ts +++ b/tests/playwright-test/reporter-raw.spec.ts @@ -110,6 +110,181 @@ test('should save attachments', async ({ runInlineTest }, testInfo) => { expect(path2).toBe('dummy-path'); }); +test(`testInfo.attach should save attachments via path`, async ({ runInlineTest }, testInfo) => { + await runInlineTest({ + 'a.test.js': ` + const path = require('path'); + const fs = require('fs'); + const { test } = pwt; + test('infer contentType from path', async ({}, testInfo) => { + const tmpPath = testInfo.outputPath('example.json'); + await fs.promises.writeFile(tmpPath, 'We <3 Playwright!'); + await testInfo.attach(tmpPath); + // Forcibly remove the tmp file to ensure attach is actually automagically copying it + await fs.promises.unlink(tmpPath); + }); + + test('infer contentType from name (over extension)', async ({}, testInfo) => { + const tmpPath = testInfo.outputPath('example.json'); + await fs.promises.writeFile(tmpPath, 'We <3 Playwright!'); + await testInfo.attach(tmpPath, { name: 'example.png' }); + // Forcibly remove the tmp file to ensure attach is actually automagically copying it + await fs.promises.unlink(tmpPath); + }); + + test('explicit contentType (over extension)', async ({}, testInfo) => { + const tmpPath = testInfo.outputPath('example.json'); + await fs.promises.writeFile(tmpPath, 'We <3 Playwright!'); + await testInfo.attach(tmpPath, { contentType: 'image/png' }); + // Forcibly remove the tmp file to ensure attach is actually automagically copying it + await fs.promises.unlink(tmpPath); + }); + + test('explicit contentType (over extension and name)', async ({}, testInfo) => { + const tmpPath = testInfo.outputPath('example.json'); + await fs.promises.writeFile(tmpPath, 'We <3 Playwright!'); + await testInfo.attach(tmpPath, { name: 'example.png', contentType: 'x-playwright/custom' }); + // Forcibly remove the tmp file to ensure attach is actually automagically copying it + await fs.promises.unlink(tmpPath); + }); + + test('fallback contentType', async ({}, testInfo) => { + const tmpPath = testInfo.outputPath('example.json'); + await fs.promises.writeFile(tmpPath, 'We <3 Playwright!'); + await testInfo.attach(tmpPath, { name: 'example.this-extension-better-not-map-to-an-actual-mimetype' }); + // Forcibly remove the tmp file to ensure attach is actually automagically copying it + await fs.promises.unlink(tmpPath); + }); + `, + }, { reporter: 'dot,' + kRawReporterPath, workers: 1 }, {}, { usesCustomOutputDir: true }); + const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8')); + { + const result = json.suites[0].tests[0].results[0]; + expect(result.attachments[0].name).toBe('example.json'); + expect(result.attachments[0].contentType).toBe('application/json'); + const p = result.attachments[0].path; + expect(p).toMatch(/[/\\]attachments[/\\]01a5667d100fac2200bf40cf43083fae0580c58e\.json$/); + const contents = fs.readFileSync(p); + expect(contents.toString()).toBe('We <3 Playwright!'); + } + { + const result = json.suites[0].tests[1].results[0]; + expect(result.attachments[0].name).toBe('example.png'); + expect(result.attachments[0].contentType).toBe('image/png'); + const p = result.attachments[0].path; + expect(p).toMatch(/[/\\]attachments[/\\]01a5667d100fac2200bf40cf43083fae0580c58e\.json$/); + const contents = fs.readFileSync(p); + expect(contents.toString()).toBe('We <3 Playwright!'); + } + { + const result = json.suites[0].tests[2].results[0]; + expect(result.attachments[0].name).toBe('example.json'); + expect(result.attachments[0].contentType).toBe('image/png'); + const p = result.attachments[0].path; + expect(p).toMatch(/[/\\]attachments[/\\]01a5667d100fac2200bf40cf43083fae0580c58e\.json$/); + const contents = fs.readFileSync(p); + expect(contents.toString()).toBe('We <3 Playwright!'); + } + { + const result = json.suites[0].tests[3].results[0]; + expect(result.attachments[0].name).toBe('example.png'); + expect(result.attachments[0].contentType).toBe('x-playwright/custom'); + const p = result.attachments[0].path; + expect(p).toMatch(/[/\\]attachments[/\\]01a5667d100fac2200bf40cf43083fae0580c58e\.json$/); + const contents = fs.readFileSync(p); + expect(contents.toString()).toBe('We <3 Playwright!'); + } + { + const result = json.suites[0].tests[4].results[0]; + expect(result.attachments[0].name).toBe('example.this-extension-better-not-map-to-an-actual-mimetype'); + expect(result.attachments[0].contentType).toBe('application/octet-stream'); + const p = result.attachments[0].path; + expect(p).toMatch(/[/\\]attachments[/\\]01a5667d100fac2200bf40cf43083fae0580c58e\.json$/); + const contents = fs.readFileSync(p); + expect(contents.toString()).toBe('We <3 Playwright!'); + } +}); + +test(`testInfo.attach should save attachments via inline attachment`, async ({ runInlineTest }, testInfo) => { + await runInlineTest({ + 'a.test.js': ` + const path = require('path'); + const fs = require('fs'); + const { test } = pwt; + test('infer contentType - string', async ({}, testInfo) => { + await testInfo.attach('We <3 Playwright!', 'example.json'); + }); + + test('infer contentType - Buffer', async ({}, testInfo) => { + await testInfo.attach(Buffer.from('We <3 Playwright!'), 'example.json'); + }); + + test('fallback contentType - string', async ({}, testInfo) => { + await testInfo.attach('We <3 Playwright!', 'example.this-extension-better-not-map-to-an-actual-mimetype'); + }); + + test('fallback contentType - Buffer', async ({}, testInfo) => { + await testInfo.attach(Buffer.from('We <3 Playwright!'), 'example.this-extension-better-not-map-to-an-actual-mimetype'); + }); + + test('fallback contentType - no extension', async ({}, testInfo) => { + await testInfo.attach('We <3 Playwright!', 'example'); + }); + + test('explicit contentType - string', async ({}, testInfo) => { + await testInfo.attach('We <3 Playwright!', 'example.json', { contentType: 'x-playwright/custom' }); + }); + + test('explicit contentType - Buffer', async ({}, testInfo) => { + await testInfo.attach(Buffer.from('We <3 Playwright!'), 'example.json', { contentType: 'x-playwright/custom' }); + }); + `, + }, { reporter: 'dot,' + kRawReporterPath, workers: 1 }, {}, { usesCustomOutputDir: true }); + const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8')); + { + const result = json.suites[0].tests[0].results[0]; + expect(result.attachments[0].name).toBe('example.json'); + expect(result.attachments[0].contentType).toBe('application/json'); + expect(Buffer.from(result.attachments[0].body, 'base64')).toEqual(Buffer.from('We <3 Playwright!')); + } + { + const result = json.suites[0].tests[1].results[0]; + expect(result.attachments[0].name).toBe('example.json'); + expect(result.attachments[0].contentType).toBe('application/json'); + expect(Buffer.from(result.attachments[0].body, 'base64')).toEqual(Buffer.from('We <3 Playwright!')); + } + { + const result = json.suites[0].tests[2].results[0]; + expect(result.attachments[0].name).toBe('example.this-extension-better-not-map-to-an-actual-mimetype'); + expect(result.attachments[0].contentType).toBe('text/plain'); + expect(Buffer.from(result.attachments[0].body, 'base64')).toEqual(Buffer.from('We <3 Playwright!')); + } + { + const result = json.suites[0].tests[3].results[0]; + expect(result.attachments[0].name).toBe('example.this-extension-better-not-map-to-an-actual-mimetype'); + expect(result.attachments[0].contentType).toBe('application/octet-stream'); + expect(Buffer.from(result.attachments[0].body, 'base64')).toEqual(Buffer.from('We <3 Playwright!')); + } + { + const result = json.suites[0].tests[4].results[0]; + expect(result.attachments[0].name).toBe('example'); + expect(result.attachments[0].contentType).toBe('text/plain'); + expect(Buffer.from(result.attachments[0].body, 'base64')).toEqual(Buffer.from('We <3 Playwright!')); + } + { + const result = json.suites[0].tests[5].results[0]; + expect(result.attachments[0].name).toBe('example.json'); + expect(result.attachments[0].contentType).toBe('x-playwright/custom'); + expect(Buffer.from(result.attachments[0].body, 'base64')).toEqual(Buffer.from('We <3 Playwright!')); + } + { + const result = json.suites[0].tests[6].results[0]; + expect(result.attachments[0].name).toBe('example.json'); + expect(result.attachments[0].contentType).toBe('x-playwright/custom'); + expect(Buffer.from(result.attachments[0].body, 'base64')).toEqual(Buffer.from('We <3 Playwright!')); + } +}); + test('dupe project names', async ({ runInlineTest }, testInfo) => { await runInlineTest({ 'playwright.config.ts': ` diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index cc40faac9b..17e3408353 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -204,6 +204,8 @@ export interface TestInfo { timeout: number; annotations: { type: string, description?: string }[]; attachments: { name: string, path?: string, body?: Buffer, contentType: string }[]; + attach(path: string, options?: { contentType?: string, name?: string}): Promise; + attach(body: string | Buffer, name: string, options?: { contentType?: string }): Promise; repeatEachIndex: number; retry: number; duration: number;