diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index 43b8474abe..4423606498 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -50,6 +50,16 @@ Start time of this particular test step. List of steps inside this step. +## property: TestStep.attachments +* since: v1.10 +- type: <[Array]<[Object]>> + - `name` <[string]> Attachment name. + - `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`. + - `path` ?<[string]> Optional path on the filesystem to the attached file. + - `body` ?<[Buffer]> Optional attachment body used instead of a file. + +The list of files or buffers attached in the step execution through [`property: TestInfo.attachments`]. + ## property: TestStep.title * since: v1.10 - type: <[string]> diff --git a/packages/html-reporter/src/testCaseView.spec.tsx b/packages/html-reporter/src/testCaseView.spec.tsx index b7a9f9405b..7cc9f8991c 100644 --- a/packages/html-reporter/src/testCaseView.spec.tsx +++ b/packages/html-reporter/src/testCaseView.spec.tsx @@ -37,8 +37,10 @@ const result: TestResult = { duration: 10, location: { file: 'test.spec.ts', line: 82, column: 0 }, steps: [], + attachments: [], count: 1, }], + attachments: [], }], attachments: [], status: 'passed', @@ -139,6 +141,7 @@ const resultWithAttachment: TestResult = { location: { file: 'test.spec.ts', line: 62, column: 0 }, count: 1, steps: [], + attachments: [1], }], attachments: [{ name: 'first attachment', diff --git a/packages/html-reporter/src/types.ts b/packages/html-reporter/src/types.ts index 733e88e8b9..7a99184739 100644 --- a/packages/html-reporter/src/types.ts +++ b/packages/html-reporter/src/types.ts @@ -108,5 +108,6 @@ export type TestStep = { snippet?: string; error?: string; steps: TestStep[]; + attachments: number[]; count: number; }; diff --git a/packages/playwright/src/common/globals.ts b/packages/playwright/src/common/globals.ts index 746e1ec847..f24a55fc60 100644 --- a/packages/playwright/src/common/globals.ts +++ b/packages/playwright/src/common/globals.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { TestInfoImpl } from '../worker/testInfo'; +import type { TestInfoImpl, TestStepInternal } from '../worker/testInfo'; import type { Suite } from './test'; let currentTestInfoValue: TestInfoImpl | null = null; @@ -42,3 +42,13 @@ export function setIsWorkerProcess() { export function isWorkerProcess() { return _isWorkerProcess; } + +let currentStepValue: TestStepInternal | undefined; + +export function setCurrentStep(step: TestStepInternal | undefined) { + currentStepValue = step; +} + +export function currentStep(): TestStepInternal | undefined { + return currentStepValue; +} diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 909df3dc8f..76ee996216 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -75,6 +75,7 @@ export type AttachmentPayload = { path?: string; body?: string; contentType: string; + stepId?: string; }; export type TestInfoErrorImpl = TestInfoError & { diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index f96547d427..c6e7da07a8 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -512,6 +512,7 @@ class TeleTestStep implements reporterTypes.TestStep { parent: reporterTypes.TestStep | undefined; duration: number = -1; steps: reporterTypes.TestStep[] = []; + attachments = []; private _startTime: number = 0; diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts index 0bd116e7a1..288dd18e1a 100644 --- a/packages/playwright/src/matchers/expect.ts +++ b/packages/playwright/src/matchers/expect.ts @@ -51,7 +51,7 @@ import { } from './matchers'; import { toMatchSnapshot, toHaveScreenshot, toHaveScreenshotStepTitle } from './toMatchSnapshot'; import type { Expect, ExpectMatcherState } from '../../types/test'; -import { currentTestInfo } from '../common/globals'; +import { currentTestInfo, setCurrentStep } from '../common/globals'; import { filteredStackTrace, trimLongString } from '../util'; import { expect as expectLibrary, @@ -322,8 +322,10 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler { }; const step = testInfo._addStep(stepInfo); + setCurrentStep(step); const reportStepError = (e: Error | unknown) => { + setCurrentStep(undefined); const jestError = isJestError(e) ? e : null; const error = jestError ? new ExpectError(jestError, customMessage, stackFrames) : e; if (jestError?.matcherResult.suggestedRebaseline) { @@ -338,6 +340,7 @@ class ExpectMetaInfoProxyHandler implements ProxyHandler { }; const finalizer = () => { + setCurrentStep(undefined); step.complete({}); }; diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts index 8d16f11fc2..d04be1ab69 100644 --- a/packages/playwright/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchSnapshot.ts @@ -16,7 +16,7 @@ import type { Locator, Page } from 'playwright-core'; import type { ExpectScreenshotOptions, Page as PageEx } from 'playwright-core/lib/client/page'; -import { currentTestInfo } from '../common/globals'; +import { currentStep, currentTestInfo } from '../common/globals'; import type { ImageComparatorOptions, Comparator } from 'playwright-core/lib/utils'; import { compareBuffersOrStrings, getComparator, isString, sanitizeForFilePath } from 'playwright-core/lib/utils'; import { @@ -29,7 +29,7 @@ import { colors } from 'playwright-core/lib/utilsBundle'; import fs from 'fs'; import path from 'path'; import { mime } from 'playwright-core/lib/utilsBundle'; -import type { TestInfoImpl } from '../worker/testInfo'; +import type { TestInfoImpl, TestStepInternal } from '../worker/testInfo'; import type { ExpectMatcherState } from '../../types/test'; import { matcherHint, type MatcherResult } from './matcherHint'; import type { FullProjectInternal } from '../common/config'; @@ -75,6 +75,7 @@ const NonConfigProperties: (keyof ToHaveScreenshotOptions)[] = [ class SnapshotHelper { readonly testInfo: TestInfoImpl; + readonly step: TestStepInternal; readonly attachmentBaseName: string; readonly legacyExpectedPath: string; readonly previousPath: string; @@ -91,6 +92,7 @@ class SnapshotHelper { constructor( testInfo: TestInfoImpl, + step: TestStepInternal, matcherName: string, locator: Locator | undefined, anonymousSnapshotExtension: string, @@ -182,6 +184,7 @@ class SnapshotHelper { this.comparator = getComparator(this.mimeType); this.testInfo = testInfo; + this.step = step; this.kind = this.mimeType.startsWith('image/') ? 'Screenshot' : 'Snapshot'; } @@ -224,9 +227,9 @@ class SnapshotHelper { const isWriteMissingMode = this.updateSnapshots !== 'none'; if (isWriteMissingMode) writeFileSync(this.expectedPath, actual); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); + this.step.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); writeFileSync(this.actualPath, actual); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); + this.step.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', writing actual.' : '.'}`; if (this.updateSnapshots === 'all' || this.updateSnapshots === 'changed') { /* eslint-disable no-console */ @@ -254,22 +257,22 @@ class SnapshotHelper { // Copy the expectation inside the `test-results/` folder for backwards compatibility, // so that one can upload `test-results/` directory and have all the data inside. writeFileSync(this.legacyExpectedPath, expected); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); + this.step.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath }); output.push(`\nExpected: ${colors.yellow(this.expectedPath)}`); } if (previous !== undefined) { writeFileSync(this.previousPath, previous); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-previous'), contentType: this.mimeType, path: this.previousPath }); + this.step.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-previous'), contentType: this.mimeType, path: this.previousPath }); output.push(`Previous: ${colors.yellow(this.previousPath)}`); } if (actual !== undefined) { writeFileSync(this.actualPath, actual); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); + this.step.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath }); output.push(`Received: ${colors.yellow(this.actualPath)}`); } if (diff !== undefined) { writeFileSync(this.diffPath, diff); - this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-diff'), contentType: this.mimeType, path: this.diffPath }); + this.step.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-diff'), contentType: this.mimeType, path: this.diffPath }); output.push(` Diff: ${colors.yellow(this.diffPath)}`); } @@ -293,7 +296,8 @@ export function toMatchSnapshot( optOptions: ImageComparatorOptions = {} ): MatcherResult { const testInfo = currentTestInfo(); - if (!testInfo) + const step = currentStep(); + if (!testInfo || !step) throw new Error(`toMatchSnapshot() must be called during the test`); if (received instanceof Promise) throw new Error('An unresolved Promise was passed to toMatchSnapshot(), make sure to resolve it by adding await to it.'); @@ -303,7 +307,7 @@ export function toMatchSnapshot( const configOptions = testInfo._projectInternal.expect?.toMatchSnapshot || {}; const helper = new SnapshotHelper( - testInfo, 'toMatchSnapshot', undefined, determineFileExtension(received), + testInfo, step, 'toMatchSnapshot', undefined, determineFileExtension(received), configOptions, nameOrOptions, optOptions); if (this.isNot) { @@ -365,7 +369,8 @@ export async function toHaveScreenshot( optOptions: ToHaveScreenshotOptions = {} ): Promise> { const testInfo = currentTestInfo(); - if (!testInfo) + const step = currentStep(); + if (!testInfo || !step) throw new Error(`toHaveScreenshot() must be called during the test`); if (testInfo._projectInternal.ignoreSnapshots) @@ -374,7 +379,7 @@ export async function toHaveScreenshot( expectTypes(pageOrLocator, ['Page', 'Locator'], 'toHaveScreenshot'); const [page, locator] = pageOrLocator.constructor.name === 'Page' ? [(pageOrLocator as PageEx), undefined] : [(pageOrLocator as Locator).page() as PageEx, pageOrLocator as Locator]; const configOptions = testInfo._projectInternal.expect?.toHaveScreenshot || {}; - const helper = new SnapshotHelper(testInfo, 'toHaveScreenshot', locator, 'png', configOptions, nameOrOptions, optOptions); + const helper = new SnapshotHelper(testInfo, step, 'toHaveScreenshot', locator, 'png', configOptions, nameOrOptions, optOptions); if (!helper.expectedPath.toLowerCase().endsWith('.png')) throw new Error(`Screenshot name "${path.basename(helper.expectedPath)}" must have '.png' extension`); expectTypes(pageOrLocator, ['Page', 'Locator'], 'toHaveScreenshot'); diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 302d532641..410b91ef87 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -505,7 +505,7 @@ class HtmlBuilder { duration: result.duration, startTime: result.startTime.toISOString(), retry: result.retry, - steps: dedupeSteps(result.steps).map(s => this._createTestStep(s)), + steps: dedupeSteps(result.steps).map(s => this._createTestStep(s, result)), errors: formatResultFailure(test, result, '', true).map(error => error.message), status: result.status, attachments: this._serializeAttachments([ @@ -515,20 +515,26 @@ class HtmlBuilder { }; } - private _createTestStep(dedupedStep: DedupedStep): TestStep { + private _createTestStep(dedupedStep: DedupedStep, result: TestResultPublic): TestStep { const { step, duration, count } = dedupedStep; - const result: TestStep = { + const testStep: TestStep = { title: step.title, startTime: step.startTime.toISOString(), duration, - steps: dedupeSteps(step.steps).map(s => this._createTestStep(s)), + steps: dedupeSteps(step.steps).map(s => this._createTestStep(s, result)), + attachments: step.attachments.map(s => { + const index = result.attachments.indexOf(s); + if (index === -1) + throw new Error('Unexpected, attachment not found'); + return index; + }), location: this._relativeLocation(step.location), error: step.error?.message, count }; if (step.location) - this._stepsInFile.set(step.location.file, result); - return result; + this._stepsInFile.set(step.location.file, testStep); + return testStep; } private _relativeLocation(location: Location | undefined): Location | undefined { diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 98e0ec1546..0dc46287c4 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -320,6 +320,7 @@ class JobDispatcher { startTime: new Date(params.wallTime), duration: -1, steps: [], + attachments: [], location: params.location, }; steps.set(params.stepId, step); @@ -361,6 +362,8 @@ class JobDispatcher { body: params.body !== undefined ? Buffer.from(params.body, 'base64') : undefined }; data.result.attachments.push(attachment); + if (params.stepId) + data.steps.get(params.stepId)!.attachments.push(attachment); } private _failTestWithErrors(test: TestCase, errors: TestError[]) { diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 8b965e0a14..c45cfc3523 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -31,7 +31,8 @@ import type { StackFrame } from '@protocol/channels'; import { testInfoError } from './util'; export interface TestStepInternal { - complete(result: { error?: Error | unknown, attachments?: Attachment[], suggestedRebaseline?: string }): void; + complete(result: { error?: Error | unknown, suggestedRebaseline?: string }): void; + attachments: Attachment[]; stepId: string; title: string; category: 'hook' | 'fixture' | 'test.step' | 'expect' | 'attach' | string; @@ -193,7 +194,7 @@ export class TestInfoImpl implements TestInfo { this._attachmentsPush = this.attachments.push.bind(this.attachments); this.attachments.push = (...attachments: TestInfo['attachments']) => { for (const a of attachments) - this._attach(a.name, a); + this._attach(a, this._parentStep()?.stepId); return this.attachments.length; }; @@ -238,7 +239,12 @@ export class TestInfoImpl implements TestInfo { } } - _addStep(data: Omit, parentStep?: TestStepInternal): TestStepInternal { + _parentStep() { + return zones.zoneData('stepZone') + ?? this._findLastStageStep(this._steps); // If no parent step on stack, assume the current stage as parent. + } + + _addStep(data: Omit, parentStep?: TestStepInternal): TestStepInternal { const stepId = `${data.category}@${++this._lastStepId}`; if (data.isStage) { @@ -246,11 +252,7 @@ export class TestInfoImpl implements TestInfo { parentStep = this._findLastStageStep(this._steps); } else { if (!parentStep) - parentStep = zones.zoneData('stepZone'); - if (!parentStep) { - // If no parent step on stack, assume the current stage as parent. - parentStep = this._findLastStageStep(this._steps); - } + parentStep = this._parentStep(); } const filteredStack = filteredStackTrace(captureRawStack()); @@ -261,10 +263,12 @@ export class TestInfoImpl implements TestInfo { } data.location = data.location || filteredStack[0]; + const attachments: Attachment[] = []; const step: TestStepInternal = { stepId, ...data, steps: [], + attachments, complete: result => { if (step.endWallTime) return; @@ -292,6 +296,9 @@ export class TestInfoImpl implements TestInfo { } } + for (const attachment of attachments ?? []) + this._attach(attachment, stepId); + const payload: StepEndPayload = { testId: this.testId, stepId, @@ -301,7 +308,7 @@ export class TestInfoImpl implements TestInfo { }; this._onStepEnd(payload); const errorForTrace = step.error ? { name: '', message: step.error.message || '', stack: step.error.stack } : undefined; - this._tracing.appendAfterActionForStep(stepId, errorForTrace, result.attachments); + this._tracing.appendAfterActionForStep(stepId, errorForTrace, attachments); } }; const parentStepList = parentStep ? parentStep.steps : this._steps; @@ -400,23 +407,24 @@ export class TestInfoImpl implements TestInfo { // ------------ TestInfo methods ------------ async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) { - this._attach(name, await normalizeAndSaveAttachment(this.outputPath(), name, options)); - } - - private _attach(name: string, attachment: TestInfo['attachments'][0]) { const step = this._addStep({ title: `attach "${name}"`, category: 'attach', }); + step.attachments.push(await normalizeAndSaveAttachment(this.outputPath(), name, options)); + step.complete({}); + } + + private _attach(attachment: TestInfo['attachments'][0], stepId: string | undefined) { this._attachmentsPush(attachment); this._onAttach({ testId: this.testId, name: attachment.name, contentType: attachment.contentType, path: attachment.path, - body: attachment.body?.toString('base64') + body: attachment.body?.toString('base64'), + stepId, }); - step.complete({ attachments: [attachment] }); } outputPath(...pathSegments: string[]){ diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 04cf03287f..d13b297a88 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -691,6 +691,33 @@ export interface TestStep { */ titlePath(): Array; + /** + * The list of files or buffers attached in the step execution through + * [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments). + */ + attachments: Array<{ + /** + * Attachment name. + */ + name: string; + + /** + * Content type of this attachment to properly present in the report, for example `'application/json'` or + * `'image/png'`. + */ + contentType: string; + + /** + * Optional path on the filesystem to the attached file. + */ + path?: string; + + /** + * Optional attachment body used instead of a file. + */ + body?: Buffer; + }>; + /** * Step category to differentiate steps with different origin and verbosity. Built-in categories are: * - `hook` for fixtures and hooks initialization and teardown diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index f036e3e494..d91702620a 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import type { Reporter, TestCase, TestResult, TestStep } from '../../packages/playwright-test/reporter'; import { test, expect } from './playwright-test-fixtures'; const smallReporterJS = ` @@ -703,3 +704,33 @@ onEnd onExit `); }); + +test('step attachments are referentially equal to result attachments', async ({ runInlineTest }) => { + class TestReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + console.log('%%%', JSON.stringify({ + title: step.title, + attachments: step.attachments.map(a => result.attachments.indexOf(a)), + })); + } + } + const result = await runInlineTest({ + 'reporter.ts': `module.exports = ${TestReporter.toString()}`, + 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({}, testInfo) => { + await test.step('step', async () => { + testInfo.attachments.push({ name: 'attachment', body: Buffer.from('content') }); + }); + }); + `, + }, { 'reporter': '', 'workers': 1 }); + + const steps = result.outputLines.map(line => JSON.parse(line)); + expect(steps).toEqual([ + { title: 'Before Hooks', attachments: [] }, + { title: 'step', attachments: [0] }, + { title: 'After Hooks', attachments: [] }, + ]); +});