diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index 43b8474abe..ef16e4849a 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -50,6 +50,16 @@ Start time of this particular test step. List of steps inside this step. +## property: TestStep.attachments +* since: v1.50 +- type: <[Array]<[Object]>> + - `name` <[string]> Attachment name. + - `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. + - `path` ?<[string]> Optional path on the filesystem to the attached file. + - `body` ?<[Buffer]> Optional attachment body used instead of a file. + +The list of files or buffers attached in the step execution through [`method: TestInfo.attach`]. + ## property: TestStep.title * since: v1.10 - type: <[string]> diff --git a/packages/html-reporter/src/links.tsx b/packages/html-reporter/src/links.tsx index b8db4c0e9e..5f199568b5 100644 --- a/packages/html-reporter/src/links.tsx +++ b/packages/html-reporter/src/links.tsx @@ -68,11 +68,12 @@ export const ProjectLink: React.FunctionComponent<{ export const AttachmentLink: React.FunctionComponent<{ attachment: TestAttachment, + result: TestResult, href?: string, linkName?: string, openInNewTab?: boolean, -}> = ({ attachment, href, linkName, openInNewTab }) => { - const isAnchored = useIsAnchored('attachment-' + attachment.name); +}> = ({ attachment, result, href, linkName, openInNewTab }) => { + const isAnchored = useIsAnchored('attachment-' + result.attachments.indexOf(attachment)); return {attachment.contentType === kMissingContentType ? icons.warning() : icons.attachment()} {attachment.path && {linkName || attachment.name}} diff --git a/packages/html-reporter/src/testCaseView.spec.tsx b/packages/html-reporter/src/testCaseView.spec.tsx index b7a9f9405b..7cc9f8991c 100644 --- a/packages/html-reporter/src/testCaseView.spec.tsx +++ b/packages/html-reporter/src/testCaseView.spec.tsx @@ -37,8 +37,10 @@ const result: TestResult = { duration: 10, location: { file: 'test.spec.ts', line: 82, column: 0 }, steps: [], + attachments: [], count: 1, }], + attachments: [], }], attachments: [], status: 'passed', @@ -139,6 +141,7 @@ const resultWithAttachment: TestResult = { location: { file: 'test.spec.ts', line: 62, column: 0 }, count: 1, steps: [], + attachments: [1], }], attachments: [{ name: 'first attachment', diff --git a/packages/html-reporter/src/testFileView.tsx b/packages/html-reporter/src/testFileView.tsx index f8fad1d646..00ea004136 100644 --- a/packages/html-reporter/src/testFileView.tsx +++ b/packages/html-reporter/src/testFileView.tsx @@ -75,7 +75,7 @@ function imageDiffBadge(test: TestCaseSummary): JSX.Element | undefined { for (const result of test.results) { for (const attachment of result.attachments) { if (attachment.contentType.startsWith('image/') && !!attachment.name.match(/-(expected|actual|diff)/)) - return {image()}; + return {image()}; } } } diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx index 410677cb02..9dcdf29092 100644 --- a/packages/html-reporter/src/testResultView.tsx +++ b/packages/html-reporter/src/testResultView.tsx @@ -32,7 +32,7 @@ interface ImageDiffWithAnchors extends ImageDiff { anchors: string[]; } -function groupImageDiffs(screenshots: Set): ImageDiffWithAnchors[] { +function groupImageDiffs(screenshots: Set, result: TestResult): ImageDiffWithAnchors[] { const snapshotNameToImageDiff = new Map(); for (const attachment of screenshots) { const match = attachment.name.match(/^(.*)-(expected|actual|diff|previous)(\.[^.]+)?$/); @@ -45,7 +45,7 @@ function groupImageDiffs(screenshots: Set): ImageDiffWithAnchors imageDiff = { name: snapshotName, anchors: [`attachment-${name}`] }; snapshotNameToImageDiff.set(snapshotName, imageDiff); } - imageDiff.anchors.push(`attachment-${attachment.name}`); + imageDiff.anchors.push(`attachment-${result.attachments.indexOf(attachment)}`); if (category === 'actual') imageDiff.actual = { attachment }; if (category === 'expected') @@ -72,15 +72,15 @@ export const TestResultView: React.FC<{ result: TestResult, }> = ({ test, result }) => { const { screenshots, videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors } = React.useMemo(() => { - const attachments = result?.attachments || []; + const attachments = result.attachments; const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/'))); - const screenshotAnchors = [...screenshots].map(a => `attachment-${a.name}`); + const screenshotAnchors = [...screenshots].map(a => `attachment-${attachments.indexOf(a)}`); const videos = attachments.filter(a => a.contentType.startsWith('video/')); const traces = attachments.filter(a => a.name === 'trace'); const otherAttachments = new Set(attachments); [...screenshots, ...videos, ...traces].forEach(a => otherAttachments.delete(a)); - const otherAttachmentAnchors = [...otherAttachments].map(a => `attachment-${a.name}`); - const diffs = groupImageDiffs(screenshots); + const otherAttachmentAnchors = [...otherAttachments].map(a => `attachment-${attachments.indexOf(a)}`); + const diffs = groupImageDiffs(screenshots, result); const errors = classifyErrors(result.errors, diffs); return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors }; }, [result]); @@ -107,11 +107,11 @@ export const TestResultView: React.FC<{ {!!screenshots.length && {screenshots.map((a, i) => { - return + return - + ; })} } @@ -121,7 +121,7 @@ export const TestResultView: React.FC<{ - {traces.map((a, i) => )} + {traces.map((a, i) => )} } } @@ -130,14 +130,14 @@ export const TestResultView: React.FC<{ - + )} } {!!otherAttachments.size && {[...otherAttachments].map((a, i) => - - + + )} } @@ -174,10 +174,9 @@ const StepTreeItem: React.FC<{ step: TestStep; depth: number, }> = ({ test, step, result, depth }) => { - const attachmentName = step.title.match(/^attach "(.*)"$/)?.[1]; return {msToString(step.duration)} - {attachmentName && { evt.stopPropagation(); }}>{icons.attachment()}} + {step.attachments.length > 0 && { evt.stopPropagation(); }}>{icons.attachment()}} {statusIcon(step.error || step.duration === -1 ? 'failed' : 'passed')} {step.title} {step.count > 1 && <> ✕ {step.count}} diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.ts index 733e88e8b9..7a99184739 100644 --- a/packages/html-reporter/src/types.d.ts +++ b/packages/html-reporter/src/types.d.ts @@ -108,5 +108,6 @@ export type TestStep = { snippet?: string; error?: string; steps: TestStep[]; + attachments: number[]; count: number; }; diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 909df3dc8f..76ee996216 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -75,6 +75,7 @@ export type AttachmentPayload = { path?: string; body?: string; contentType: string; + stepId?: string; }; export type TestInfoErrorImpl = TestInfoError & { diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index f96547d427..1d41b793cd 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -108,6 +108,7 @@ export type JsonTestStepEnd = { id: string; duration: number; error?: reporterTypes.TestError; + attachments?: number[]; // index of JsonTestResultEnd.attachments }; export type JsonFullResult = { @@ -249,7 +250,7 @@ export class TeleReporterReceiver { const parentStep = payload.parentStepId ? result._stepMap.get(payload.parentStepId) : undefined; const location = this._absoluteLocation(payload.location); - const step = new TeleTestStep(payload, parentStep, location); + const step = new TeleTestStep(payload, parentStep, location, result); if (parentStep) parentStep.steps.push(step); else @@ -262,6 +263,7 @@ export class TeleReporterReceiver { const test = this._tests.get(testId)!; const result = test.results.find(r => r._id === resultId)!; const step = result._stepMap.get(payload.id)!; + step._endPayload = payload; step.duration = payload.duration; step.error = payload.error; this._reporter.onStepEnd?.(test, result, step); @@ -512,15 +514,20 @@ class TeleTestStep implements reporterTypes.TestStep { parent: reporterTypes.TestStep | undefined; duration: number = -1; steps: reporterTypes.TestStep[] = []; + error: reporterTypes.TestError | undefined; + + private _result: TeleTestResult; + _endPayload?: JsonTestStepEnd; private _startTime: number = 0; - constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined) { + constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined, result: TeleTestResult) { this.title = payload.title; this.category = payload.category; this.location = location; this.parent = parentStep; this._startTime = payload.startTime; + this._result = result; } titlePath() { @@ -535,6 +542,10 @@ class TeleTestStep implements reporterTypes.TestStep { set startTime(value: Date) { this._startTime = +value; } + + get attachments() { + return this._endPayload?.attachments?.map(index => this._result.attachments[index]) ?? []; + } } export class TeleTestResult implements reporterTypes.TestResult { @@ -550,7 +561,7 @@ export class TeleTestResult implements reporterTypes.TestResult { errors: reporterTypes.TestResult['errors'] = []; error: reporterTypes.TestResult['error']; - _stepMap: Map = new Map(); + _stepMap = new Map(); _id: string; private _startTime: number = 0; diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 75d345e319..62158eef6d 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -505,7 +505,7 @@ class HtmlBuilder { duration: result.duration, startTime: result.startTime.toISOString(), retry: result.retry, - steps: dedupeSteps(result.steps).map(s => this._createTestStep(s)), + steps: dedupeSteps(result.steps).map(s => this._createTestStep(s, result)), errors: formatResultFailure(test, result, '', true).map(error => error.message), status: result.status, attachments: this._serializeAttachments([ @@ -515,20 +515,26 @@ class HtmlBuilder { }; } - private _createTestStep(dedupedStep: DedupedStep): TestStep { + private _createTestStep(dedupedStep: DedupedStep, result: api.TestResult): TestStep { const { step, duration, count } = dedupedStep; - const result: TestStep = { + const testStep: TestStep = { title: step.title, startTime: step.startTime.toISOString(), duration, - steps: dedupeSteps(step.steps).map(s => this._createTestStep(s)), + steps: dedupeSteps(step.steps).map(s => this._createTestStep(s, result)), + attachments: step.attachments.map(s => { + const index = result.attachments.indexOf(s); + if (index === -1) + throw new Error('Unexpected, attachment not found'); + return index; + }), location: this._relativeLocation(step.location), error: step.error?.message, count }; if (step.location) - this._stepsInFile.set(step.location.file, result); - return result; + this._stepsInFile.set(step.location.file, testStep); + return testStep; } private _relativeLocation(location: api.Location | undefined): api.Location | undefined { diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index f56178114d..0ec92ae9ac 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -100,7 +100,7 @@ export class TeleReporterEmitter implements ReporterV2 { params: { testId: test.id, resultId: (result as any)[this._idSymbol], - step: this._serializeStepEnd(step) + step: this._serializeStepEnd(step, result) } }); } @@ -251,11 +251,12 @@ export class TeleReporterEmitter implements ReporterV2 { }; } - private _serializeStepEnd(step: reporterTypes.TestStep): teleReceiver.JsonTestStepEnd { + private _serializeStepEnd(step: reporterTypes.TestStep, result: reporterTypes.TestResult): teleReceiver.JsonTestStepEnd { return { id: (step as any)[this._idSymbol], duration: step.duration, error: step.error, + attachments: step.attachments.map(a => result.attachments.indexOf(a)), }; } diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 98e0ec1546..534fe7eb4a 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -320,6 +320,7 @@ class JobDispatcher { startTime: new Date(params.wallTime), duration: -1, steps: [], + attachments: [], location: params.location, }; steps.set(params.stepId, step); @@ -361,6 +362,13 @@ class JobDispatcher { body: params.body !== undefined ? Buffer.from(params.body, 'base64') : undefined }; data.result.attachments.push(attachment); + if (params.stepId) { + const step = data.steps.get(params.stepId); + if (step) + step.attachments.push(attachment); + else + this._reporter.onStdErr?.('Internal error: step id not found: ' + params.stepId); + } } private _failTestWithErrors(test: TestCase, errors: TestError[]) { diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 8b965e0a14..6577e19d0d 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -17,6 +17,7 @@ import fs from 'fs'; import path from 'path'; import { captureRawStack, monotonicTime, zones, sanitizeForFilePath, stringifyStackFrames } from 'playwright-core/lib/utils'; +import type { ExpectZone } from 'playwright-core/lib/utils'; import type { TestInfo, TestStatus, FullProject } from '../../types/test'; import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc'; import type { TestCase } from '../common/test'; @@ -26,12 +27,12 @@ import type { Annotation, FullConfigInternal, FullProjectInternal } from '../com import type { FullConfig, Location } from '../../types/testReporter'; import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, trimLongString, windowsFilesystemFriendlyLength } from '../util'; import { TestTracing } from './testTracing'; -import type { Attachment } from './testTracing'; import type { StackFrame } from '@protocol/channels'; import { testInfoError } from './util'; export interface TestStepInternal { - complete(result: { error?: Error | unknown, attachments?: Attachment[], suggestedRebaseline?: string }): void; + complete(result: { error?: Error | unknown, suggestedRebaseline?: string }): void; + attachmentIndices: number[]; stepId: string; title: string; category: 'hook' | 'fixture' | 'test.step' | 'expect' | 'attach' | string; @@ -69,6 +70,7 @@ export class TestInfoImpl implements TestInfo { readonly _projectInternal: FullProjectInternal; readonly _configInternal: FullConfigInternal; private readonly _steps: TestStepInternal[] = []; + private readonly _stepMap = new Map(); _onDidFinishTestFunction: (() => Promise) | undefined; _hasNonRetriableError = false; _hasUnhandledError = false; @@ -193,7 +195,7 @@ export class TestInfoImpl implements TestInfo { this._attachmentsPush = this.attachments.push.bind(this.attachments); this.attachments.push = (...attachments: TestInfo['attachments']) => { for (const a of attachments) - this._attach(a.name, a); + this._attach(a, this._expectStepId() ?? this._parentStep()?.stepId); return this.attachments.length; }; @@ -238,7 +240,16 @@ export class TestInfoImpl implements TestInfo { } } - _addStep(data: Omit, parentStep?: TestStepInternal): TestStepInternal { + private _parentStep() { + return zones.zoneData('stepZone') + ?? this._findLastStageStep(this._steps); // If no parent step on stack, assume the current stage as parent. + } + + private _expectStepId() { + return zones.zoneData('expectZone')?.stepId; + } + + _addStep(data: Omit, parentStep?: TestStepInternal): TestStepInternal { const stepId = `${data.category}@${++this._lastStepId}`; if (data.isStage) { @@ -246,11 +257,7 @@ export class TestInfoImpl implements TestInfo { parentStep = this._findLastStageStep(this._steps); } else { if (!parentStep) - parentStep = zones.zoneData('stepZone'); - if (!parentStep) { - // If no parent step on stack, assume the current stage as parent. - parentStep = this._findLastStageStep(this._steps); - } + parentStep = this._parentStep(); } const filteredStack = filteredStackTrace(captureRawStack()); @@ -261,10 +268,12 @@ export class TestInfoImpl implements TestInfo { } data.location = data.location || filteredStack[0]; + const attachmentIndices: number[] = []; const step: TestStepInternal = { stepId, ...data, steps: [], + attachmentIndices, complete: result => { if (step.endWallTime) return; @@ -301,11 +310,13 @@ export class TestInfoImpl implements TestInfo { }; this._onStepEnd(payload); const errorForTrace = step.error ? { name: '', message: step.error.message || '', stack: step.error.stack } : undefined; - this._tracing.appendAfterActionForStep(stepId, errorForTrace, result.attachments); + const attachments = attachmentIndices.map(i => this.attachments[i]); + this._tracing.appendAfterActionForStep(stepId, errorForTrace, attachments); } }; const parentStepList = parentStep ? parentStep.steps : this._steps; parentStepList.push(step); + this._stepMap.set(stepId, step); const payload: StepBeginPayload = { testId: this.testId, stepId, @@ -400,23 +411,33 @@ export class TestInfoImpl implements TestInfo { // ------------ TestInfo methods ------------ async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) { - this._attach(name, await normalizeAndSaveAttachment(this.outputPath(), name, options)); - } - - private _attach(name: string, attachment: TestInfo['attachments'][0]) { const step = this._addStep({ title: `attach "${name}"`, category: 'attach', }); - this._attachmentsPush(attachment); + this._attach(await normalizeAndSaveAttachment(this.outputPath(), name, options), step.stepId); + step.complete({}); + } + + private _attach(attachment: TestInfo['attachments'][0], stepId: string | undefined) { + const index = this._attachmentsPush(attachment) - 1; + if (stepId) { + this._stepMap.get(stepId)!.attachmentIndices.push(index); + } else { + // trace viewer has no means of representing attachments outside of a step, so we create an artificial action + const callId = `attach@${++this._lastStepId}`; + this._tracing.appendBeforeActionForStep(callId, this._findLastStageStep(this._steps)?.stepId, `attach "${attachment.name}"`, undefined, []); + this._tracing.appendAfterActionForStep(callId, undefined, [attachment]); + } + this._onAttach({ testId: this.testId, name: attachment.name, contentType: attachment.contentType, path: attachment.path, - body: attachment.body?.toString('base64') + body: attachment.body?.toString('base64'), + stepId, }); - step.complete({ attachments: [attachment] }); } outputPath(...pathSegments: string[]){ diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 04cf03287f..3f3a43984e 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -691,6 +691,33 @@ export interface TestStep { */ titlePath(): Array; + /** + * The list of files or buffers attached in the step execution through + * [testInfo.attach(name[, options])](https://playwright.dev/docs/api/class-testinfo#test-info-attach). + */ + attachments: Array<{ + /** + * Attachment name. + */ + name: string; + + /** + * Content type of this attachment to properly present in the report, for example `'application/json'` or + * `'image/png'`. + */ + contentType: string; + + /** + * Optional path on the filesystem to the attached file. + */ + path?: string; + + /** + * Optional attachment body used instead of a file. + */ + body?: Buffer; + }>; + /** * Step category to differentiate steps with different origin and verbosity. Built-in categories are: * - `hook` for fixtures and hooks initialization and teardown diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts index 5c5d6c304a..9f06e8f3b4 100644 --- a/tests/playwright-test/playwright.trace.spec.ts +++ b/tests/playwright-test/playwright.trace.spec.ts @@ -540,7 +540,7 @@ test('should include attachments by default', async ({ runInlineTest, server }, contentType: 'text/plain', sha1: expect.any(String), }]); - expect([...trace.resources.keys()].filter(f => f.startsWith('resources/'))).toHaveLength(1); + expect([...trace.resources.keys()]).toContain(`resources/${trace.actions[1].attachments[0].sha1}`); }); test('should opt out of attachments', async ({ runInlineTest, server }, testInfo) => { @@ -566,7 +566,7 @@ test('should opt out of attachments', async ({ runInlineTest, server }, testInfo 'After Hooks', ]); expect(trace.actions[1].attachments).toEqual(undefined); - expect([...trace.resources.keys()].filter(f => f.startsWith('resources/'))).toHaveLength(0); + expect([...trace.resources.keys()].filter(f => f.startsWith('resources/') && !f.startsWith('resources/src@'))).toHaveLength(0); }); test('should record with custom page fixture', async ({ runInlineTest }, testInfo) => { @@ -761,7 +761,7 @@ test('should not throw when screenshot on failure fails', async ({ runInlineTest expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); const trace = await parseTrace(testInfo.outputPath('test-results', 'a-has-download-page', 'trace.zip')); - const attachedScreenshots = trace.actionTree.filter(s => s.trim() === `attach "screenshot"`); + const attachedScreenshots = trace.actions.flatMap(a => a.attachments); // One screenshot for the page, no screenshot for the download page since it should have failed. expect(attachedScreenshots.length).toBe(1); }); diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 556d12e8a2..7c7c60836f 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -941,6 +941,32 @@ for (const useIntermediateMergeReport of [true, false] as const) { await expect(attachment).toBeInViewport(); }); + test('steps with internal attachments have links', async ({ runInlineTest, page, showReport }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('passing', async ({ page }, testInfo) => { + for (let i = 0; i < 100; i++) + await testInfo.attach('spacer', { body: 'content' }); + + await test.step('step', async () => { + testInfo.attachments.push({ name: 'attachment', body: 'content', contentType: 'text/plain' }); + }) + + }); + `, + }, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' }); + expect(result.exitCode).toBe(0); + + await showReport(); + await page.getByRole('link', { name: 'passing' }).click(); + + const attachment = page.getByText('attachment', { exact: true }); + await expect(attachment).not.toBeInViewport(); + await page.getByLabel('step').getByTitle('link to attachment').click(); + await expect(attachment).toBeInViewport(); + }); + test('should highlight textual diff', async ({ runInlineTest, showReport, page }) => { const result = await runInlineTest({ 'helper.ts': ` diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index f036e3e494..d91702620a 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import type { Reporter, TestCase, TestResult, TestStep } from '../../packages/playwright-test/reporter'; import { test, expect } from './playwright-test-fixtures'; const smallReporterJS = ` @@ -703,3 +704,33 @@ onEnd onExit `); }); + +test('step attachments are referentially equal to result attachments', async ({ runInlineTest }) => { + class TestReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + console.log('%%%', JSON.stringify({ + title: step.title, + attachments: step.attachments.map(a => result.attachments.indexOf(a)), + })); + } + } + const result = await runInlineTest({ + 'reporter.ts': `module.exports = ${TestReporter.toString()}`, + 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({}, testInfo) => { + await test.step('step', async () => { + testInfo.attachments.push({ name: 'attachment', body: Buffer.from('content') }); + }); + }); + `, + }, { 'reporter': '', 'workers': 1 }); + + const steps = result.outputLines.map(line => JSON.parse(line)); + expect(steps).toEqual([ + { title: 'Before Hooks', attachments: [] }, + { title: 'step', attachments: [0] }, + { title: 'After Hooks', attachments: [] }, + ]); +}); diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts index 83642bd19e..3afa7a8d90 100644 --- a/tests/playwright-test/to-have-screenshot.spec.ts +++ b/tests/playwright-test/to-have-screenshot.spec.ts @@ -263,11 +263,7 @@ test('should report toHaveScreenshot step with expectation name in title', async `end browserContext.newPage`, `end fixture: page`, `end Before Hooks`, - `end attach "foo-expected.png"`, - `end attach "foo-actual.png"`, `end expect.toHaveScreenshot(foo.png)`, - `end attach "is-a-test-1-expected.png"`, - `end attach "is-a-test-1-actual.png"`, `end expect.toHaveScreenshot(is-a-test-1.png)`, `end fixture: page`, `end fixture: context`, @@ -681,6 +677,30 @@ test('should write missing expectations locally twice and attach them', async ({ ]); }); +test('should attach missing expectations to right step', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + onStepEnd(test, result, step) { + if (step.attachments.length > 0) + console.log(\`%%\${step.title}: \${step.attachments.map(a => a.name).join(", ")}\`); + } + } + module.exports = Reporter; + `, + ...playwrightConfig({ reporter: [['dot'], ['./reporter']] }), + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png'); + }); + `, + }, { reporter: '' }); + + expect(result.exitCode).toBe(1); + expect(result.outputLines).toEqual(['expect.toHaveScreenshot(snapshot.png): snapshot-expected.png, snapshot-actual.png']); +}); + test('shouldn\'t write missing expectations locally for negated matcher', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ ...playwrightConfig({ diff --git a/tests/playwright-test/ui-mode-trace.spec.ts b/tests/playwright-test/ui-mode-trace.spec.ts index 06cff62399..23a321338b 100644 --- a/tests/playwright-test/ui-mode-trace.spec.ts +++ b/tests/playwright-test/ui-mode-trace.spec.ts @@ -94,8 +94,6 @@ test('should merge screenshot assertions', async ({ runUITest }, testInfo) => { /Before Hooks[\d.]+m?s/, /page.setContent[\d.]+m?s/, /expect.toHaveScreenshot[\d.]+m?s/, - /attach "trace-test-1-expected.png/, - /attach "trace-test-1-actual.png/, /After Hooks[\d.]+m?s/, /Worker Cleanup[\d.]+m?s/, ]); @@ -425,3 +423,50 @@ test('should show custom fixture titles in actions tree', async ({ runUITest }) /After Hooks[\d.]+m?s/, ]); }); + +test('attachments tab shows all but top-level .push attachments', async ({ runUITest }) => { + const { page } = await runUITest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('attachment test', async ({}) => { + await test.step('step', async () => { + test.info().attachments.push({ + name: 'foo-push', + body: Buffer.from('foo-content'), + contentType: 'text/plain' + }); + + await test.info().attach('foo-attach', { body: 'foo-content' }) + }); + + test.info().attachments.push({ + name: 'bar-push', + body: Buffer.from('bar-content'), + contentType: 'text/plain' + }); + await test.info().attach('bar-attach', { body: 'bar-content' }) + }); + `, + }); + + await page.getByRole('treeitem', { name: 'attachment test' }).dblclick(); + const actionsTree = page.getByTestId('actions-tree'); + await actionsTree.getByRole('treeitem', { name: 'step' }).click(); + await page.keyboard.press('ArrowRight'); + await expect(actionsTree, 'attach() and top-level attachments.push calls are shown as actions').toMatchAriaSnapshot(` + - tree: + - treeitem /step/: + - group: + - treeitem /attach \\"foo-attach\\"/ + - treeitem /attach \\"bar-push\\"/ + - treeitem /attach \\"bar-attach\\"/ + `); + await page.getByRole('tab', { name: 'Attachments' }).click(); + await expect(page.getByRole('tabpanel', { name: 'Attachments' })).toMatchAriaSnapshot(` + - tabpanel: + - button /foo-push/ + - button /foo-attach/ + - button /bar-push/ + - button /bar-attach/ + `); +});