feat(reporter): report TestStep#attachments (#34037)

This commit is contained in:
Simon Knott 2025-01-02 17:48:59 +01:00 committed by GitHub
parent 175f05cafc
commit 04a3574f80
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 265 additions and 54 deletions

View file

@ -50,6 +50,16 @@ Start time of this particular test step.
List of steps inside this step. List of steps inside this step.
## property: TestStep.attachments
* since: v1.50
- type: <[Array]<[Object]>>
- `name` <[string]> Attachment name.
- `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`.
- `path` ?<[string]> Optional path on the filesystem to the attached file.
- `body` ?<[Buffer]> Optional attachment body used instead of a file.
The list of files or buffers attached in the step execution through [`method: TestInfo.attach`].
## property: TestStep.title ## property: TestStep.title
* since: v1.10 * since: v1.10
- type: <[string]> - type: <[string]>

View file

@ -68,11 +68,12 @@ export const ProjectLink: React.FunctionComponent<{
export const AttachmentLink: React.FunctionComponent<{ export const AttachmentLink: React.FunctionComponent<{
attachment: TestAttachment, attachment: TestAttachment,
result: TestResult,
href?: string, href?: string,
linkName?: string, linkName?: string,
openInNewTab?: boolean, openInNewTab?: boolean,
}> = ({ attachment, href, linkName, openInNewTab }) => { }> = ({ attachment, result, href, linkName, openInNewTab }) => {
const isAnchored = useIsAnchored('attachment-' + attachment.name); const isAnchored = useIsAnchored('attachment-' + result.attachments.indexOf(attachment));
return <TreeItem title={<span> return <TreeItem title={<span>
{attachment.contentType === kMissingContentType ? icons.warning() : icons.attachment()} {attachment.contentType === kMissingContentType ? icons.warning() : icons.attachment()}
{attachment.path && <a href={href || attachment.path} download={downloadFileNameForAttachment(attachment)}>{linkName || attachment.name}</a>} {attachment.path && <a href={href || attachment.path} download={downloadFileNameForAttachment(attachment)}>{linkName || attachment.name}</a>}

View file

@ -37,8 +37,10 @@ const result: TestResult = {
duration: 10, duration: 10,
location: { file: 'test.spec.ts', line: 82, column: 0 }, location: { file: 'test.spec.ts', line: 82, column: 0 },
steps: [], steps: [],
attachments: [],
count: 1, count: 1,
}], }],
attachments: [],
}], }],
attachments: [], attachments: [],
status: 'passed', status: 'passed',
@ -139,6 +141,7 @@ const resultWithAttachment: TestResult = {
location: { file: 'test.spec.ts', line: 62, column: 0 }, location: { file: 'test.spec.ts', line: 62, column: 0 },
count: 1, count: 1,
steps: [], steps: [],
attachments: [1],
}], }],
attachments: [{ attachments: [{
name: 'first attachment', name: 'first attachment',

View file

@ -75,7 +75,7 @@ function imageDiffBadge(test: TestCaseSummary): JSX.Element | undefined {
for (const result of test.results) { for (const result of test.results) {
for (const attachment of result.attachments) { for (const attachment of result.attachments) {
if (attachment.contentType.startsWith('image/') && !!attachment.name.match(/-(expected|actual|diff)/)) if (attachment.contentType.startsWith('image/') && !!attachment.name.match(/-(expected|actual|diff)/))
return <Link href={testResultHref({ test, result, anchor: `attachment-${attachment.name}` })} title='View images' className='test-file-badge'>{image()}</Link>; return <Link href={testResultHref({ test, result, anchor: `attachment-${result.attachments.indexOf(attachment)}` })} title='View images' className='test-file-badge'>{image()}</Link>;
} }
} }
} }

View file

@ -32,7 +32,7 @@ interface ImageDiffWithAnchors extends ImageDiff {
anchors: string[]; anchors: string[];
} }
function groupImageDiffs(screenshots: Set<TestAttachment>): ImageDiffWithAnchors[] { function groupImageDiffs(screenshots: Set<TestAttachment>, result: TestResult): ImageDiffWithAnchors[] {
const snapshotNameToImageDiff = new Map<string, ImageDiffWithAnchors>(); const snapshotNameToImageDiff = new Map<string, ImageDiffWithAnchors>();
for (const attachment of screenshots) { for (const attachment of screenshots) {
const match = attachment.name.match(/^(.*)-(expected|actual|diff|previous)(\.[^.]+)?$/); const match = attachment.name.match(/^(.*)-(expected|actual|diff|previous)(\.[^.]+)?$/);
@ -45,7 +45,7 @@ function groupImageDiffs(screenshots: Set<TestAttachment>): ImageDiffWithAnchors
imageDiff = { name: snapshotName, anchors: [`attachment-${name}`] }; imageDiff = { name: snapshotName, anchors: [`attachment-${name}`] };
snapshotNameToImageDiff.set(snapshotName, imageDiff); snapshotNameToImageDiff.set(snapshotName, imageDiff);
} }
imageDiff.anchors.push(`attachment-${attachment.name}`); imageDiff.anchors.push(`attachment-${result.attachments.indexOf(attachment)}`);
if (category === 'actual') if (category === 'actual')
imageDiff.actual = { attachment }; imageDiff.actual = { attachment };
if (category === 'expected') if (category === 'expected')
@ -72,15 +72,15 @@ export const TestResultView: React.FC<{
result: TestResult, result: TestResult,
}> = ({ test, result }) => { }> = ({ test, result }) => {
const { screenshots, videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors } = React.useMemo(() => { const { screenshots, videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors } = React.useMemo(() => {
const attachments = result?.attachments || []; const attachments = result.attachments;
const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/'))); const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/')));
const screenshotAnchors = [...screenshots].map(a => `attachment-${a.name}`); const screenshotAnchors = [...screenshots].map(a => `attachment-${attachments.indexOf(a)}`);
const videos = attachments.filter(a => a.contentType.startsWith('video/')); const videos = attachments.filter(a => a.contentType.startsWith('video/'));
const traces = attachments.filter(a => a.name === 'trace'); const traces = attachments.filter(a => a.name === 'trace');
const otherAttachments = new Set<TestAttachment>(attachments); const otherAttachments = new Set<TestAttachment>(attachments);
[...screenshots, ...videos, ...traces].forEach(a => otherAttachments.delete(a)); [...screenshots, ...videos, ...traces].forEach(a => otherAttachments.delete(a));
const otherAttachmentAnchors = [...otherAttachments].map(a => `attachment-${a.name}`); const otherAttachmentAnchors = [...otherAttachments].map(a => `attachment-${attachments.indexOf(a)}`);
const diffs = groupImageDiffs(screenshots); const diffs = groupImageDiffs(screenshots, result);
const errors = classifyErrors(result.errors, diffs); const errors = classifyErrors(result.errors, diffs);
return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors }; return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors };
}, [result]); }, [result]);
@ -107,11 +107,11 @@ export const TestResultView: React.FC<{
{!!screenshots.length && <AutoChip header='Screenshots' revealOnAnchorId={screenshotAnchors}> {!!screenshots.length && <AutoChip header='Screenshots' revealOnAnchorId={screenshotAnchors}>
{screenshots.map((a, i) => { {screenshots.map((a, i) => {
return <Anchor key={`screenshot-${i}`} id={`attachment-${a.name}`}> return <Anchor key={`screenshot-${i}`} id={`attachment-${result.attachments.indexOf(a)}`}>
<a href={a.path}> <a href={a.path}>
<img className='screenshot' src={a.path} /> <img className='screenshot' src={a.path} />
</a> </a>
<AttachmentLink attachment={a}></AttachmentLink> <AttachmentLink attachment={a} result={result}></AttachmentLink>
</Anchor>; </Anchor>;
})} })}
</AutoChip>} </AutoChip>}
@ -121,7 +121,7 @@ export const TestResultView: React.FC<{
<a href={generateTraceUrl(traces)}> <a href={generateTraceUrl(traces)}>
<img className='screenshot' src={traceImage} style={{ width: 192, height: 117, marginLeft: 20 }} /> <img className='screenshot' src={traceImage} style={{ width: 192, height: 117, marginLeft: 20 }} />
</a> </a>
{traces.map((a, i) => <AttachmentLink key={`trace-${i}`} attachment={a} linkName={traces.length === 1 ? 'trace' : `trace-${i + 1}`}></AttachmentLink>)} {traces.map((a, i) => <AttachmentLink key={`trace-${i}`} attachment={a} result={result} linkName={traces.length === 1 ? 'trace' : `trace-${i + 1}`}></AttachmentLink>)}
</div>} </div>}
</AutoChip></Anchor>} </AutoChip></Anchor>}
@ -130,14 +130,14 @@ export const TestResultView: React.FC<{
<video controls> <video controls>
<source src={a.path} type={a.contentType}/> <source src={a.path} type={a.contentType}/>
</video> </video>
<AttachmentLink attachment={a}></AttachmentLink> <AttachmentLink attachment={a} result={result}></AttachmentLink>
</div>)} </div>)}
</AutoChip></Anchor>} </AutoChip></Anchor>}
{!!otherAttachments.size && <AutoChip header='Attachments' revealOnAnchorId={otherAttachmentAnchors}> {!!otherAttachments.size && <AutoChip header='Attachments' revealOnAnchorId={otherAttachmentAnchors}>
{[...otherAttachments].map((a, i) => {[...otherAttachments].map((a, i) =>
<Anchor key={`attachment-link-${i}`} id={`attachment-${a.name}`}> <Anchor key={`attachment-link-${i}`} id={`attachment-${result.attachments.indexOf(a)}`}>
<AttachmentLink attachment={a} openInNewTab={a.contentType.startsWith('text/html')} /> <AttachmentLink attachment={a} result={result} openInNewTab={a.contentType.startsWith('text/html')} />
</Anchor> </Anchor>
)} )}
</AutoChip>} </AutoChip>}
@ -174,10 +174,9 @@ const StepTreeItem: React.FC<{
step: TestStep; step: TestStep;
depth: number, depth: number,
}> = ({ test, step, result, depth }) => { }> = ({ test, step, result, depth }) => {
const attachmentName = step.title.match(/^attach "(.*)"$/)?.[1];
return <TreeItem title={<span aria-label={step.title}> return <TreeItem title={<span aria-label={step.title}>
<span style={{ float: 'right' }}>{msToString(step.duration)}</span> <span style={{ float: 'right' }}>{msToString(step.duration)}</span>
{attachmentName && <a style={{ float: 'right' }} title='link to attachment' href={testResultHref({ test, result, anchor: `attachment-${attachmentName}` })} onClick={evt => { evt.stopPropagation(); }}>{icons.attachment()}</a>} {step.attachments.length > 0 && <a style={{ float: 'right' }} title='link to attachment' href={testResultHref({ test, result, anchor: `attachment-${step.attachments[0]}` })} onClick={evt => { evt.stopPropagation(); }}>{icons.attachment()}</a>}
{statusIcon(step.error || step.duration === -1 ? 'failed' : 'passed')} {statusIcon(step.error || step.duration === -1 ? 'failed' : 'passed')}
<span>{step.title}</span> <span>{step.title}</span>
{step.count > 1 && <> <span className='test-result-counter'>{step.count}</span></>} {step.count > 1 && <> <span className='test-result-counter'>{step.count}</span></>}

View file

@ -108,5 +108,6 @@ export type TestStep = {
snippet?: string; snippet?: string;
error?: string; error?: string;
steps: TestStep[]; steps: TestStep[];
attachments: number[];
count: number; count: number;
}; };

View file

@ -75,6 +75,7 @@ export type AttachmentPayload = {
path?: string; path?: string;
body?: string; body?: string;
contentType: string; contentType: string;
stepId?: string;
}; };
export type TestInfoErrorImpl = TestInfoError & { export type TestInfoErrorImpl = TestInfoError & {

View file

@ -108,6 +108,7 @@ export type JsonTestStepEnd = {
id: string; id: string;
duration: number; duration: number;
error?: reporterTypes.TestError; error?: reporterTypes.TestError;
attachments?: number[]; // index of JsonTestResultEnd.attachments
}; };
export type JsonFullResult = { export type JsonFullResult = {
@ -249,7 +250,7 @@ export class TeleReporterReceiver {
const parentStep = payload.parentStepId ? result._stepMap.get(payload.parentStepId) : undefined; const parentStep = payload.parentStepId ? result._stepMap.get(payload.parentStepId) : undefined;
const location = this._absoluteLocation(payload.location); const location = this._absoluteLocation(payload.location);
const step = new TeleTestStep(payload, parentStep, location); const step = new TeleTestStep(payload, parentStep, location, result);
if (parentStep) if (parentStep)
parentStep.steps.push(step); parentStep.steps.push(step);
else else
@ -262,6 +263,7 @@ export class TeleReporterReceiver {
const test = this._tests.get(testId)!; const test = this._tests.get(testId)!;
const result = test.results.find(r => r._id === resultId)!; const result = test.results.find(r => r._id === resultId)!;
const step = result._stepMap.get(payload.id)!; const step = result._stepMap.get(payload.id)!;
step._endPayload = payload;
step.duration = payload.duration; step.duration = payload.duration;
step.error = payload.error; step.error = payload.error;
this._reporter.onStepEnd?.(test, result, step); this._reporter.onStepEnd?.(test, result, step);
@ -512,15 +514,20 @@ class TeleTestStep implements reporterTypes.TestStep {
parent: reporterTypes.TestStep | undefined; parent: reporterTypes.TestStep | undefined;
duration: number = -1; duration: number = -1;
steps: reporterTypes.TestStep[] = []; steps: reporterTypes.TestStep[] = [];
error: reporterTypes.TestError | undefined;
private _result: TeleTestResult;
_endPayload?: JsonTestStepEnd;
private _startTime: number = 0; private _startTime: number = 0;
constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined) { constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined, result: TeleTestResult) {
this.title = payload.title; this.title = payload.title;
this.category = payload.category; this.category = payload.category;
this.location = location; this.location = location;
this.parent = parentStep; this.parent = parentStep;
this._startTime = payload.startTime; this._startTime = payload.startTime;
this._result = result;
} }
titlePath() { titlePath() {
@ -535,6 +542,10 @@ class TeleTestStep implements reporterTypes.TestStep {
set startTime(value: Date) { set startTime(value: Date) {
this._startTime = +value; this._startTime = +value;
} }
get attachments() {
return this._endPayload?.attachments?.map(index => this._result.attachments[index]) ?? [];
}
} }
export class TeleTestResult implements reporterTypes.TestResult { export class TeleTestResult implements reporterTypes.TestResult {
@ -550,7 +561,7 @@ export class TeleTestResult implements reporterTypes.TestResult {
errors: reporterTypes.TestResult['errors'] = []; errors: reporterTypes.TestResult['errors'] = [];
error: reporterTypes.TestResult['error']; error: reporterTypes.TestResult['error'];
_stepMap: Map<string, reporterTypes.TestStep> = new Map(); _stepMap = new Map<string, TeleTestStep>();
_id: string; _id: string;
private _startTime: number = 0; private _startTime: number = 0;

View file

@ -505,7 +505,7 @@ class HtmlBuilder {
duration: result.duration, duration: result.duration,
startTime: result.startTime.toISOString(), startTime: result.startTime.toISOString(),
retry: result.retry, retry: result.retry,
steps: dedupeSteps(result.steps).map(s => this._createTestStep(s)), steps: dedupeSteps(result.steps).map(s => this._createTestStep(s, result)),
errors: formatResultFailure(test, result, '', true).map(error => error.message), errors: formatResultFailure(test, result, '', true).map(error => error.message),
status: result.status, status: result.status,
attachments: this._serializeAttachments([ attachments: this._serializeAttachments([
@ -515,20 +515,26 @@ class HtmlBuilder {
}; };
} }
private _createTestStep(dedupedStep: DedupedStep): TestStep { private _createTestStep(dedupedStep: DedupedStep, result: api.TestResult): TestStep {
const { step, duration, count } = dedupedStep; const { step, duration, count } = dedupedStep;
const result: TestStep = { const testStep: TestStep = {
title: step.title, title: step.title,
startTime: step.startTime.toISOString(), startTime: step.startTime.toISOString(),
duration, duration,
steps: dedupeSteps(step.steps).map(s => this._createTestStep(s)), steps: dedupeSteps(step.steps).map(s => this._createTestStep(s, result)),
attachments: step.attachments.map(s => {
const index = result.attachments.indexOf(s);
if (index === -1)
throw new Error('Unexpected, attachment not found');
return index;
}),
location: this._relativeLocation(step.location), location: this._relativeLocation(step.location),
error: step.error?.message, error: step.error?.message,
count count
}; };
if (step.location) if (step.location)
this._stepsInFile.set(step.location.file, result); this._stepsInFile.set(step.location.file, testStep);
return result; return testStep;
} }
private _relativeLocation(location: api.Location | undefined): api.Location | undefined { private _relativeLocation(location: api.Location | undefined): api.Location | undefined {

View file

@ -100,7 +100,7 @@ export class TeleReporterEmitter implements ReporterV2 {
params: { params: {
testId: test.id, testId: test.id,
resultId: (result as any)[this._idSymbol], resultId: (result as any)[this._idSymbol],
step: this._serializeStepEnd(step) step: this._serializeStepEnd(step, result)
} }
}); });
} }
@ -251,11 +251,12 @@ export class TeleReporterEmitter implements ReporterV2 {
}; };
} }
private _serializeStepEnd(step: reporterTypes.TestStep): teleReceiver.JsonTestStepEnd { private _serializeStepEnd(step: reporterTypes.TestStep, result: reporterTypes.TestResult): teleReceiver.JsonTestStepEnd {
return { return {
id: (step as any)[this._idSymbol], id: (step as any)[this._idSymbol],
duration: step.duration, duration: step.duration,
error: step.error, error: step.error,
attachments: step.attachments.map(a => result.attachments.indexOf(a)),
}; };
} }

View file

@ -320,6 +320,7 @@ class JobDispatcher {
startTime: new Date(params.wallTime), startTime: new Date(params.wallTime),
duration: -1, duration: -1,
steps: [], steps: [],
attachments: [],
location: params.location, location: params.location,
}; };
steps.set(params.stepId, step); steps.set(params.stepId, step);
@ -361,6 +362,13 @@ class JobDispatcher {
body: params.body !== undefined ? Buffer.from(params.body, 'base64') : undefined body: params.body !== undefined ? Buffer.from(params.body, 'base64') : undefined
}; };
data.result.attachments.push(attachment); data.result.attachments.push(attachment);
if (params.stepId) {
const step = data.steps.get(params.stepId);
if (step)
step.attachments.push(attachment);
else
this._reporter.onStdErr?.('Internal error: step id not found: ' + params.stepId);
}
} }
private _failTestWithErrors(test: TestCase, errors: TestError[]) { private _failTestWithErrors(test: TestCase, errors: TestError[]) {

View file

@ -17,6 +17,7 @@
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { captureRawStack, monotonicTime, zones, sanitizeForFilePath, stringifyStackFrames } from 'playwright-core/lib/utils'; import { captureRawStack, monotonicTime, zones, sanitizeForFilePath, stringifyStackFrames } from 'playwright-core/lib/utils';
import type { ExpectZone } from 'playwright-core/lib/utils';
import type { TestInfo, TestStatus, FullProject } from '../../types/test'; import type { TestInfo, TestStatus, FullProject } from '../../types/test';
import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc'; import type { AttachmentPayload, StepBeginPayload, StepEndPayload, TestInfoErrorImpl, WorkerInitParams } from '../common/ipc';
import type { TestCase } from '../common/test'; import type { TestCase } from '../common/test';
@ -26,12 +27,12 @@ import type { Annotation, FullConfigInternal, FullProjectInternal } from '../com
import type { FullConfig, Location } from '../../types/testReporter'; import type { FullConfig, Location } from '../../types/testReporter';
import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, trimLongString, windowsFilesystemFriendlyLength } from '../util'; import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, trimLongString, windowsFilesystemFriendlyLength } from '../util';
import { TestTracing } from './testTracing'; import { TestTracing } from './testTracing';
import type { Attachment } from './testTracing';
import type { StackFrame } from '@protocol/channels'; import type { StackFrame } from '@protocol/channels';
import { testInfoError } from './util'; import { testInfoError } from './util';
export interface TestStepInternal { export interface TestStepInternal {
complete(result: { error?: Error | unknown, attachments?: Attachment[], suggestedRebaseline?: string }): void; complete(result: { error?: Error | unknown, suggestedRebaseline?: string }): void;
attachmentIndices: number[];
stepId: string; stepId: string;
title: string; title: string;
category: 'hook' | 'fixture' | 'test.step' | 'expect' | 'attach' | string; category: 'hook' | 'fixture' | 'test.step' | 'expect' | 'attach' | string;
@ -69,6 +70,7 @@ export class TestInfoImpl implements TestInfo {
readonly _projectInternal: FullProjectInternal; readonly _projectInternal: FullProjectInternal;
readonly _configInternal: FullConfigInternal; readonly _configInternal: FullConfigInternal;
private readonly _steps: TestStepInternal[] = []; private readonly _steps: TestStepInternal[] = [];
private readonly _stepMap = new Map<string, TestStepInternal>();
_onDidFinishTestFunction: (() => Promise<void>) | undefined; _onDidFinishTestFunction: (() => Promise<void>) | undefined;
_hasNonRetriableError = false; _hasNonRetriableError = false;
_hasUnhandledError = false; _hasUnhandledError = false;
@ -193,7 +195,7 @@ export class TestInfoImpl implements TestInfo {
this._attachmentsPush = this.attachments.push.bind(this.attachments); this._attachmentsPush = this.attachments.push.bind(this.attachments);
this.attachments.push = (...attachments: TestInfo['attachments']) => { this.attachments.push = (...attachments: TestInfo['attachments']) => {
for (const a of attachments) for (const a of attachments)
this._attach(a.name, a); this._attach(a, this._expectStepId() ?? this._parentStep()?.stepId);
return this.attachments.length; return this.attachments.length;
}; };
@ -238,7 +240,16 @@ export class TestInfoImpl implements TestInfo {
} }
} }
_addStep(data: Omit<TestStepInternal, 'complete' | 'stepId' | 'steps'>, parentStep?: TestStepInternal): TestStepInternal { private _parentStep() {
return zones.zoneData<TestStepInternal>('stepZone')
?? this._findLastStageStep(this._steps); // If no parent step on stack, assume the current stage as parent.
}
private _expectStepId() {
return zones.zoneData<ExpectZone>('expectZone')?.stepId;
}
_addStep(data: Omit<TestStepInternal, 'complete' | 'stepId' | 'steps' | 'attachmentIndices'>, parentStep?: TestStepInternal): TestStepInternal {
const stepId = `${data.category}@${++this._lastStepId}`; const stepId = `${data.category}@${++this._lastStepId}`;
if (data.isStage) { if (data.isStage) {
@ -246,11 +257,7 @@ export class TestInfoImpl implements TestInfo {
parentStep = this._findLastStageStep(this._steps); parentStep = this._findLastStageStep(this._steps);
} else { } else {
if (!parentStep) if (!parentStep)
parentStep = zones.zoneData<TestStepInternal>('stepZone'); parentStep = this._parentStep();
if (!parentStep) {
// If no parent step on stack, assume the current stage as parent.
parentStep = this._findLastStageStep(this._steps);
}
} }
const filteredStack = filteredStackTrace(captureRawStack()); const filteredStack = filteredStackTrace(captureRawStack());
@ -261,10 +268,12 @@ export class TestInfoImpl implements TestInfo {
} }
data.location = data.location || filteredStack[0]; data.location = data.location || filteredStack[0];
const attachmentIndices: number[] = [];
const step: TestStepInternal = { const step: TestStepInternal = {
stepId, stepId,
...data, ...data,
steps: [], steps: [],
attachmentIndices,
complete: result => { complete: result => {
if (step.endWallTime) if (step.endWallTime)
return; return;
@ -301,11 +310,13 @@ export class TestInfoImpl implements TestInfo {
}; };
this._onStepEnd(payload); this._onStepEnd(payload);
const errorForTrace = step.error ? { name: '', message: step.error.message || '', stack: step.error.stack } : undefined; const errorForTrace = step.error ? { name: '', message: step.error.message || '', stack: step.error.stack } : undefined;
this._tracing.appendAfterActionForStep(stepId, errorForTrace, result.attachments); const attachments = attachmentIndices.map(i => this.attachments[i]);
this._tracing.appendAfterActionForStep(stepId, errorForTrace, attachments);
} }
}; };
const parentStepList = parentStep ? parentStep.steps : this._steps; const parentStepList = parentStep ? parentStep.steps : this._steps;
parentStepList.push(step); parentStepList.push(step);
this._stepMap.set(stepId, step);
const payload: StepBeginPayload = { const payload: StepBeginPayload = {
testId: this.testId, testId: this.testId,
stepId, stepId,
@ -400,23 +411,33 @@ export class TestInfoImpl implements TestInfo {
// ------------ TestInfo methods ------------ // ------------ TestInfo methods ------------
async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) { async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) {
this._attach(name, await normalizeAndSaveAttachment(this.outputPath(), name, options));
}
private _attach(name: string, attachment: TestInfo['attachments'][0]) {
const step = this._addStep({ const step = this._addStep({
title: `attach "${name}"`, title: `attach "${name}"`,
category: 'attach', category: 'attach',
}); });
this._attachmentsPush(attachment); this._attach(await normalizeAndSaveAttachment(this.outputPath(), name, options), step.stepId);
step.complete({});
}
private _attach(attachment: TestInfo['attachments'][0], stepId: string | undefined) {
const index = this._attachmentsPush(attachment) - 1;
if (stepId) {
this._stepMap.get(stepId)!.attachmentIndices.push(index);
} else {
// trace viewer has no means of representing attachments outside of a step, so we create an artificial action
const callId = `attach@${++this._lastStepId}`;
this._tracing.appendBeforeActionForStep(callId, this._findLastStageStep(this._steps)?.stepId, `attach "${attachment.name}"`, undefined, []);
this._tracing.appendAfterActionForStep(callId, undefined, [attachment]);
}
this._onAttach({ this._onAttach({
testId: this.testId, testId: this.testId,
name: attachment.name, name: attachment.name,
contentType: attachment.contentType, contentType: attachment.contentType,
path: attachment.path, path: attachment.path,
body: attachment.body?.toString('base64') body: attachment.body?.toString('base64'),
stepId,
}); });
step.complete({ attachments: [attachment] });
} }
outputPath(...pathSegments: string[]){ outputPath(...pathSegments: string[]){

View file

@ -691,6 +691,33 @@ export interface TestStep {
*/ */
titlePath(): Array<string>; titlePath(): Array<string>;
/**
* The list of files or buffers attached in the step execution through
* [testInfo.attach(name[, options])](https://playwright.dev/docs/api/class-testinfo#test-info-attach).
*/
attachments: Array<{
/**
* Attachment name.
*/
name: string;
/**
* Content type of this attachment to properly present in the report, for example `'application/json'` or
* `'image/png'`.
*/
contentType: string;
/**
* Optional path on the filesystem to the attached file.
*/
path?: string;
/**
* Optional attachment body used instead of a file.
*/
body?: Buffer;
}>;
/** /**
* Step category to differentiate steps with different origin and verbosity. Built-in categories are: * Step category to differentiate steps with different origin and verbosity. Built-in categories are:
* - `hook` for fixtures and hooks initialization and teardown * - `hook` for fixtures and hooks initialization and teardown

View file

@ -540,7 +540,7 @@ test('should include attachments by default', async ({ runInlineTest, server },
contentType: 'text/plain', contentType: 'text/plain',
sha1: expect.any(String), sha1: expect.any(String),
}]); }]);
expect([...trace.resources.keys()].filter(f => f.startsWith('resources/'))).toHaveLength(1); expect([...trace.resources.keys()]).toContain(`resources/${trace.actions[1].attachments[0].sha1}`);
}); });
test('should opt out of attachments', async ({ runInlineTest, server }, testInfo) => { test('should opt out of attachments', async ({ runInlineTest, server }, testInfo) => {
@ -566,7 +566,7 @@ test('should opt out of attachments', async ({ runInlineTest, server }, testInfo
'After Hooks', 'After Hooks',
]); ]);
expect(trace.actions[1].attachments).toEqual(undefined); expect(trace.actions[1].attachments).toEqual(undefined);
expect([...trace.resources.keys()].filter(f => f.startsWith('resources/'))).toHaveLength(0); expect([...trace.resources.keys()].filter(f => f.startsWith('resources/') && !f.startsWith('resources/src@'))).toHaveLength(0);
}); });
test('should record with custom page fixture', async ({ runInlineTest }, testInfo) => { test('should record with custom page fixture', async ({ runInlineTest }, testInfo) => {
@ -761,7 +761,7 @@ test('should not throw when screenshot on failure fails', async ({ runInlineTest
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1); expect(result.passed).toBe(1);
const trace = await parseTrace(testInfo.outputPath('test-results', 'a-has-download-page', 'trace.zip')); const trace = await parseTrace(testInfo.outputPath('test-results', 'a-has-download-page', 'trace.zip'));
const attachedScreenshots = trace.actionTree.filter(s => s.trim() === `attach "screenshot"`); const attachedScreenshots = trace.actions.flatMap(a => a.attachments);
// One screenshot for the page, no screenshot for the download page since it should have failed. // One screenshot for the page, no screenshot for the download page since it should have failed.
expect(attachedScreenshots.length).toBe(1); expect(attachedScreenshots.length).toBe(1);
}); });

View file

@ -941,6 +941,32 @@ for (const useIntermediateMergeReport of [true, false] as const) {
await expect(attachment).toBeInViewport(); await expect(attachment).toBeInViewport();
}); });
test('steps with internal attachments have links', async ({ runInlineTest, page, showReport }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('passing', async ({ page }, testInfo) => {
for (let i = 0; i < 100; i++)
await testInfo.attach('spacer', { body: 'content' });
await test.step('step', async () => {
testInfo.attachments.push({ name: 'attachment', body: 'content', contentType: 'text/plain' });
})
});
`,
}, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' });
expect(result.exitCode).toBe(0);
await showReport();
await page.getByRole('link', { name: 'passing' }).click();
const attachment = page.getByText('attachment', { exact: true });
await expect(attachment).not.toBeInViewport();
await page.getByLabel('step').getByTitle('link to attachment').click();
await expect(attachment).toBeInViewport();
});
test('should highlight textual diff', async ({ runInlineTest, showReport, page }) => { test('should highlight textual diff', async ({ runInlineTest, showReport, page }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'helper.ts': ` 'helper.ts': `

View file

@ -14,6 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
import type { Reporter, TestCase, TestResult, TestStep } from '../../packages/playwright-test/reporter';
import { test, expect } from './playwright-test-fixtures'; import { test, expect } from './playwright-test-fixtures';
const smallReporterJS = ` const smallReporterJS = `
@ -703,3 +704,33 @@ onEnd
onExit onExit
`); `);
}); });
test('step attachments are referentially equal to result attachments', async ({ runInlineTest }) => {
class TestReporter implements Reporter {
onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
console.log('%%%', JSON.stringify({
title: step.title,
attachments: step.attachments.map(a => result.attachments.indexOf(a)),
}));
}
}
const result = await runInlineTest({
'reporter.ts': `module.exports = ${TestReporter.toString()}`,
'playwright.config.ts': `module.exports = { reporter: './reporter' };`,
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({}, testInfo) => {
await test.step('step', async () => {
testInfo.attachments.push({ name: 'attachment', body: Buffer.from('content') });
});
});
`,
}, { 'reporter': '', 'workers': 1 });
const steps = result.outputLines.map(line => JSON.parse(line));
expect(steps).toEqual([
{ title: 'Before Hooks', attachments: [] },
{ title: 'step', attachments: [0] },
{ title: 'After Hooks', attachments: [] },
]);
});

View file

@ -263,11 +263,7 @@ test('should report toHaveScreenshot step with expectation name in title', async
`end browserContext.newPage`, `end browserContext.newPage`,
`end fixture: page`, `end fixture: page`,
`end Before Hooks`, `end Before Hooks`,
`end attach "foo-expected.png"`,
`end attach "foo-actual.png"`,
`end expect.toHaveScreenshot(foo.png)`, `end expect.toHaveScreenshot(foo.png)`,
`end attach "is-a-test-1-expected.png"`,
`end attach "is-a-test-1-actual.png"`,
`end expect.toHaveScreenshot(is-a-test-1.png)`, `end expect.toHaveScreenshot(is-a-test-1.png)`,
`end fixture: page`, `end fixture: page`,
`end fixture: context`, `end fixture: context`,
@ -681,6 +677,30 @@ test('should write missing expectations locally twice and attach them', async ({
]); ]);
}); });
test('should attach missing expectations to right step', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': `
class Reporter {
onStepEnd(test, result, step) {
if (step.attachments.length > 0)
console.log(\`%%\${step.title}: \${step.attachments.map(a => a.name).join(", ")}\`);
}
}
module.exports = Reporter;
`,
...playwrightConfig({ reporter: [['dot'], ['./reporter']] }),
'a.spec.js': `
const { test, expect } = require('@playwright/test');
test('is a test', async ({ page }) => {
await expect(page).toHaveScreenshot('snapshot.png');
});
`,
}, { reporter: '' });
expect(result.exitCode).toBe(1);
expect(result.outputLines).toEqual(['expect.toHaveScreenshot(snapshot.png): snapshot-expected.png, snapshot-actual.png']);
});
test('shouldn\'t write missing expectations locally for negated matcher', async ({ runInlineTest }, testInfo) => { test('shouldn\'t write missing expectations locally for negated matcher', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({ const result = await runInlineTest({
...playwrightConfig({ ...playwrightConfig({

View file

@ -94,8 +94,6 @@ test('should merge screenshot assertions', async ({ runUITest }, testInfo) => {
/Before Hooks[\d.]+m?s/, /Before Hooks[\d.]+m?s/,
/page.setContent[\d.]+m?s/, /page.setContent[\d.]+m?s/,
/expect.toHaveScreenshot[\d.]+m?s/, /expect.toHaveScreenshot[\d.]+m?s/,
/attach "trace-test-1-expected.png/,
/attach "trace-test-1-actual.png/,
/After Hooks[\d.]+m?s/, /After Hooks[\d.]+m?s/,
/Worker Cleanup[\d.]+m?s/, /Worker Cleanup[\d.]+m?s/,
]); ]);
@ -425,3 +423,50 @@ test('should show custom fixture titles in actions tree', async ({ runUITest })
/After Hooks[\d.]+m?s/, /After Hooks[\d.]+m?s/,
]); ]);
}); });
test('attachments tab shows all but top-level .push attachments', async ({ runUITest }) => {
const { page } = await runUITest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('attachment test', async ({}) => {
await test.step('step', async () => {
test.info().attachments.push({
name: 'foo-push',
body: Buffer.from('foo-content'),
contentType: 'text/plain'
});
await test.info().attach('foo-attach', { body: 'foo-content' })
});
test.info().attachments.push({
name: 'bar-push',
body: Buffer.from('bar-content'),
contentType: 'text/plain'
});
await test.info().attach('bar-attach', { body: 'bar-content' })
});
`,
});
await page.getByRole('treeitem', { name: 'attachment test' }).dblclick();
const actionsTree = page.getByTestId('actions-tree');
await actionsTree.getByRole('treeitem', { name: 'step' }).click();
await page.keyboard.press('ArrowRight');
await expect(actionsTree, 'attach() and top-level attachments.push calls are shown as actions').toMatchAriaSnapshot(`
- tree:
- treeitem /step/:
- group:
- treeitem /attach \\"foo-attach\\"/
- treeitem /attach \\"bar-push\\"/
- treeitem /attach \\"bar-attach\\"/
`);
await page.getByRole('tab', { name: 'Attachments' }).click();
await expect(page.getByRole('tabpanel', { name: 'Attachments' })).toMatchAriaSnapshot(`
- tabpanel:
- button /foo-push/
- button /foo-attach/
- button /bar-push/
- button /bar-attach/
`);
});