Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix(test runner): show codeframe and location from the error top stack frame (#11179)

Browse source 
Previously, reporter would look for a stack frame directly in the test file.
Often times, that is not a top stack frame, especially when the test uses
some helper functions.

This changes error snippets and locations to use the top frame. When top
frame does not match the test file, we additionally show the location
to avoid confusion:

```
  1) a.spec.ts:7:7 › foobar ========================================================================

    Error: oh my

       at helper.ts:5

      3 |
      4 |       export function ohMy() {
    > 5 |         throw new Error('oh my');
        |               ^
      6 |       }
      7 |

        at ohMy (.../reporter-base-should-print-codeframe-from-a-helper/helper.ts:5:15)
        at .../reporter-base-should-print-codeframe-from-a-helper/a.spec.ts:8:9
        at FixtureRunner.resolveParametersAndRunHookOrTest (.../src/fixtures.ts:281:12)
```
This commit is contained in:
Dmitry Gozman 2022-01-04 16:00:55 -08:00 committed by GitHub
parent 9fcb8ace4e
commit 16a779a5ff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 104 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

124
packages/playwright-test/src/reporters/base.ts
View file

@ -14,35 +14,33 @@
* limitations under the License. * limitations under the License.
*/ */
import { BabelCodeFrameOptions, codeFrameColumns } from '@babel/code-frame'; import { codeFrameColumns } from '@babel/code-frame';
import colors from 'colors/safe'; import colors from 'colors/safe';
import fs from 'fs'; import fs from 'fs';
import milliseconds from 'ms'; import milliseconds from 'ms';
import path from 'path'; import path from 'path';
import StackUtils from 'stack-utils'; import StackUtils from 'stack-utils';
import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResult, TestStep } from '../../types/testReporter'; import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResult, TestStep, Location } from '../../types/testReporter';
const stackUtils = new StackUtils(); const stackUtils = new StackUtils();
export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' }; export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' };
export const kOutputSymbol = Symbol('output'); export const kOutputSymbol = Symbol('output');
export type PositionInFile = { column: number; line: number };
type Annotation = { type Annotation = {
filePath: string;
title: string; title: string;
message: string; message: string;
position?: PositionInFile; location?: Location;
}; };
type FailureDetails = { type FailureDetails = {
tokens: string[]; tokens: string[];
position?: PositionInFile; location?: Location;
}; };
type ErrorDetails = { type ErrorDetails = {
message: string; message: string;
position?: PositionInFile; location?: Location;
}; };
type TestSummary = { type TestSummary = {
@ -102,7 +100,7 @@ export class BaseReporter implements Reporter {
} }
onError(error: TestError) { onError(error: TestError) {
console.log(formatError(error, colors.enabled).message); console.log(formatError(this.config, error, colors.enabled).message);
} }
async onEnd(result: FullResult) { async onEnd(result: FullResult) {
@ -224,11 +222,11 @@ export class BaseReporter implements Reporter {
} }
} }
export function formatFailure(config: FullConfig, test: TestCase, options: {index?: number, includeStdio?: boolean, includeAttachments?: boolean, filePath?: string} = {}): { export function formatFailure(config: FullConfig, test: TestCase, options: {index?: number, includeStdio?: boolean, includeAttachments?: boolean} = {}): {
message: string, message: string,
annotations: Annotation[] annotations: Annotation[]
} { } {
const { index, includeStdio, includeAttachments = true, filePath } = options; const { index, includeStdio, includeAttachments = true } = options;
const lines: string[] = []; const lines: string[] = [];
const title = formatTestTitle(config, test); const title = formatTestTitle(config, test);
const annotations: Annotation[] = []; const annotations: Annotation[] = [];
@ -236,7 +234,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
lines.push(colors.red(header)); lines.push(colors.red(header));
for (const result of test.results) { for (const result of test.results) {
const resultLines: string[] = []; const resultLines: string[] = [];
const { tokens: resultTokens, position } = formatResultFailure(test, result, ' ', colors.enabled); const { tokens: resultTokens, location } = formatResultFailure(config, test, result, ' ', colors.enabled);
if (!resultTokens.length) if (!resultTokens.length)
continue; continue;
if (result.retry) { if (result.retry) {
@ -281,14 +279,11 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
resultLines.push(''); resultLines.push('');
resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-')); resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-'));
} }
if (filePath) { annotations.push({
annotations.push({ location,
filePath, title,
position, message: [header, ...resultLines].join('\n'),
title, });
message: [header, ...resultLines].join('\n'),
});
}
lines.push(...resultLines); lines.push(...resultLines);
} }
lines.push(''); lines.push('');
@ -298,7 +293,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
}; };
} }
export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): FailureDetails { export function formatResultFailure(config: FullConfig, test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): FailureDetails {
const resultTokens: string[] = []; const resultTokens: string[] = [];
if (result.status === 'timedOut') { if (result.status === 'timedOut') {
resultTokens.push(''); resultTokens.push('');
@ -310,17 +305,21 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI
} }
let error: ErrorDetails | undefined = undefined; let error: ErrorDetails | undefined = undefined;
if (result.error !== undefined) { if (result.error !== undefined) {
error = formatError(result.error, highlightCode, test.location.file); error = formatError(config, result.error, highlightCode, test.location.file);
resultTokens.push(indent(error.message, initialIndent)); resultTokens.push(indent(error.message, initialIndent));
} }
return { return {
tokens: resultTokens, tokens: resultTokens,
position: error?.position, location: error?.location,
}; };
} }
function relativeFilePath(config: FullConfig, file: string): string {
return path.relative(config.rootDir, file) || path.basename(file);
}
function relativeTestPath(config: FullConfig, test: TestCase): string { function relativeTestPath(config: FullConfig, test: TestCase): string {
return path.relative(config.rootDir, test.location.file) || path.basename(test.location.file); return relativeFilePath(config, test.location.file);
} }
function stepSuffix(step: TestStep | undefined) { function stepSuffix(step: TestStep | undefined) {
@ -342,32 +341,39 @@ function formatTestHeader(config: FullConfig, test: TestCase, indent: string, in
return pad(header, '='); return pad(header, '=');
} }
export function formatError(error: TestError, highlightCode: boolean, file?: string): ErrorDetails { export function formatError(config: FullConfig, error: TestError, highlightCode: boolean, file?: string): ErrorDetails {
const stack = error.stack; const stack = error.stack;
const tokens = ['']; const tokens = [''];
let positionInFile: PositionInFile | undefined; let location: Location | undefined;
if (stack) { if (stack) {
const { message, stackLines, position } = prepareErrorStack( const parsed = prepareErrorStack(stack);
stack, tokens.push(parsed.message);
file location = parsed.location;
); if (location) {
positionInFile = position; try {
tokens.push(message); // Stack will have /private/var/folders instead of /var/folders on Mac.
const realFile = fs.realpathSync(location.file);
const codeFrame = generateCodeFrame({ highlightCode }, file, position); const source = fs.readFileSync(realFile, 'utf8');
if (codeFrame) { const codeFrame = codeFrameColumns(source, { start: location }, { highlightCode });
tokens.push(''); if (!file || file !== realFile) {
tokens.push(codeFrame); tokens.push('');
tokens.push(colors.gray(` at `) + `${relativeFilePath(config, realFile)}:${location.line}`);
}
tokens.push('');
tokens.push(codeFrame);
} catch (e) {
// Failed to read the source file - that's ok.
}
} }
tokens.push(''); tokens.push('');
tokens.push(colors.dim(stackLines.join('\n'))); tokens.push(colors.dim(parsed.stackLines.join('\n')));
} else if (error.message) { } else if (error.message) {
tokens.push(error.message); tokens.push(error.message);
} else if (error.value) { } else if (error.value) {
tokens.push(error.value); tokens.push(error.value);
} }
return { return {
position: positionInFile, location,
message: tokens.join('\n'), message: tokens.join('\n'),
}; };
} }
@ -382,48 +388,26 @@ function indent(lines: string, tab: string) {
return lines.replace(/^(?=.+$)/gm, tab); return lines.replace(/^(?=.+$)/gm, tab);
} }
export function generateCodeFrame(options: BabelCodeFrameOptions, file?: string, position?: PositionInFile): string | undefined { export function prepareErrorStack(stack: string): {
if (!position || !file)
return;
const source = fs.readFileSync(file!, 'utf8');
const codeFrame = codeFrameColumns(
source,
{ start: position },
options
);
return codeFrame;
}
export function prepareErrorStack(stack: string, file?: string): {
message: string; message: string;
stackLines: string[]; stackLines: string[];
position?: PositionInFile; location?: Location;
} { } {
const lines = stack.split('\n'); const lines = stack.split('\n');
let firstStackLine = lines.findIndex(line => line.startsWith(' at ')); let firstStackLine = lines.findIndex(line => line.startsWith(' at '));
if (firstStackLine === -1) firstStackLine = lines.length; if (firstStackLine === -1)
firstStackLine = lines.length;
const message = lines.slice(0, firstStackLine).join('\n'); const message = lines.slice(0, firstStackLine).join('\n');
const stackLines = lines.slice(firstStackLine); const stackLines = lines.slice(firstStackLine);
const position = file ? positionInFile(stackLines, file) : undefined; let location: Location | undefined;
return {
message,
stackLines,
position,
};
}
function positionInFile(stackLines: string[], file: string): PositionInFile | undefined {
// Stack will have /private/var/folders instead of /var/folders on Mac.
file = fs.realpathSync(file);
for (const line of stackLines) { for (const line of stackLines) {
const parsed = stackUtils.parseLine(line); const parsed = stackUtils.parseLine(line);
if (!parsed || !parsed.file) if (parsed && parsed.file) {
continue; location = { file: path.join(process.cwd(), parsed.file), column: parsed.column || 0, line: parsed.line || 0 };
if (path.resolve(process.cwd(), parsed.file) === file) break;
return { column: parsed.column || 0, line: parsed.line || 0 }; }
} }
return { message, stackLines, location };
} }
function monotonicTime(): number { function monotonicTime(): number {

14
packages/playwright-test/src/reporters/github.ts
View file

@ -69,7 +69,7 @@ export class GitHubReporter extends BaseReporter {
} }
override onError(error: TestError) { override onError(error: TestError) {
const errorMessage = formatError(error, false).message; const errorMessage = formatError(this.config, error, false).message;
this.githubLogger.error(errorMessage); this.githubLogger.error(errorMessage);
} }
@ -100,21 +100,19 @@ export class GitHubReporter extends BaseReporter {
private _printFailureAnnotations(failures: TestCase[]) { private _printFailureAnnotations(failures: TestCase[]) {
failures.forEach((test, index) => { failures.forEach((test, index) => {
const filePath = workspaceRelativePath(test.location.file);
const { annotations } = formatFailure(this.config, test, { const { annotations } = formatFailure(this.config, test, {
filePath,
index: index + 1, index: index + 1,
includeStdio: true, includeStdio: true,
includeAttachments: false, includeAttachments: false,
}); });
annotations.forEach(({ filePath, title, message, position }) => { annotations.forEach(({ location, title, message }) => {
const options: GitHubLogOptions = { const options: GitHubLogOptions = {
file: filePath, file: workspaceRelativePath(location?.file || test.location.file),
title, title,
}; };
if (position) { if (location) {
options.line = position.line; options.line = location.line;
options.col = position.column; options.col = location.column;
} }
this.githubLogger.error(message, options); this.githubLogger.error(message, options);
}); });

18
packages/playwright-test/src/reporters/json.ts
View file

@ -17,7 +17,7 @@
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, TestStatus, Location, Reporter } from '../../types/testReporter'; import { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, TestStatus, Location, Reporter } from '../../types/testReporter';
import { PositionInFile, prepareErrorStack } from './base'; import { prepareErrorStack } from './base';
export interface JSONReport { export interface JSONReport {
config: Omit<FullConfig, 'projects'> & { config: Omit<FullConfig, 'projects'> & {
@ -77,7 +77,7 @@ export interface JSONReportTestResult {
body?: string; body?: string;
contentType: string; contentType: string;
}[]; }[];
errorLocation?: PositionInFile errorLocation?: Location;
} }
export interface JSONReportTestStep { export interface JSONReportTestStep {
title: string; title: string;
@ -229,12 +229,12 @@ class JSONReporter implements Reporter {
annotations: test.annotations, annotations: test.annotations,
expectedStatus: test.expectedStatus, expectedStatus: test.expectedStatus,
projectName: test.titlePath()[1], projectName: test.titlePath()[1],
results: test.results.map(r => this._serializeTestResult(r, test.location.file)), results: test.results.map(r => this._serializeTestResult(r)),
status: test.outcome(), status: test.outcome(),
}; };
} }
private _serializeTestResult(result: TestResult, file: string): JSONReportTestResult { private _serializeTestResult(result: TestResult): JSONReportTestResult {
const steps = result.steps.filter(s => s.category === 'test.step'); const steps = result.steps.filter(s => s.category === 'test.step');
const jsonResult: JSONReportTestResult = { const jsonResult: JSONReportTestResult = {
workerIndex: result.workerIndex, workerIndex: result.workerIndex,
@ -252,14 +252,8 @@ class JSONReporter implements Reporter {
body: a.body?.toString('base64') body: a.body?.toString('base64')
})), })),
}; };
if (result.error?.stack) { if (result.error?.stack)
const { position } = prepareErrorStack( jsonResult.errorLocation = prepareErrorStack(result.error.stack).location;
result.error.stack,
file
);
if (position)
jsonResult.errorLocation = position;
}
return jsonResult; return jsonResult;
} }

2
packages/playwright-test/src/reporters/raw.ts
View file

@ -219,7 +219,7 @@ class RawReporter {
startTime: result.startTime.toISOString(), startTime: result.startTime.toISOString(),
duration: result.duration, duration: result.duration,
status: result.status, status: result.status,
error: formatResultFailure(test, result, '', true).tokens.join('').trim(), error: formatResultFailure(this.config, test, result, '', true).tokens.join('').trim(),
attachments: this._createAttachments(result), attachments: this._createAttachments(result),
steps: dedupeSteps(result.steps.map(step => this._serializeStep(test, step))) steps: dedupeSteps(result.steps.map(step => this._serializeStep(test, step)))
}; };

1
tests/playwright-test/blink-diff.spec.ts
View file

@ -730,7 +730,6 @@ it.describe('Blink-Diff', () => {
it('should crop image-a', async () => { it('should crop image-a', async () => {
instance._cropImageA = { width: 1, height: 2 }; instance._cropImageA = { width: 1, height: 2 };
console.log('A');
const result = instance.runSync(); const result = instance.runSync();
expect(result.dimension).toBe(2); expect(result.dimension).toBe(2);
}); });

58
tests/playwright-test/reporter-base.spec.ts
View file

@ -64,26 +64,15 @@ test('print should print the error name without a message', async ({ runInlineTe
expect(result.output).toContain('FooBarError'); expect(result.output).toContain('FooBarError');
}); });
test('print an error in a codeframe', async ({ runInlineTest }) => { test('should print an error in a codeframe', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'my-lib.ts': `
const foobar = () => {
const error = new Error('my-message');
error.name = 'FooBarError';
throw error;
}
export default () => {
foobar();
}
`,
'a.spec.ts': ` 'a.spec.ts': `
const { test } = pwt; const { test } = pwt;
import myLib from './my-lib'; test('foobar', async ({}) => {
test('foobar', async ({}) => { const error = new Error('my-message');
const error = new Error('my-message'); error.name = 'FooBarError';
error.name = 'FooBarError'; throw error;
throw error; });
});
` `
}, {}, { }, {}, {
FORCE_COLOR: '0', FORCE_COLOR: '0',
@ -91,9 +80,36 @@ test('print an error in a codeframe', async ({ runInlineTest }) => {
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1); expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError: my-message'); expect(result.output).toContain('FooBarError: my-message');
expect(result.output).toContain('test(\'foobar\', async'); expect(result.output).not.toContain('at a.spec.ts:7');
expect(result.output).toContain('throw error;'); expect(result.output).toContain(` 5 | const { test } = pwt;`);
expect(result.output).toContain('import myLib from \'./my-lib\';'); expect(result.output).toContain(` 6 | test('foobar', async ({}) => {`);
expect(result.output).toContain(`> 7 | const error = new Error('my-message');`);
});
test('should print codeframe from a helper', async ({ runInlineTest }) => {
const result = await runInlineTest({
'helper.ts': `
export function ohMy() {
throw new Error('oh my');
}
`,
'a.spec.ts': `
import { ohMy } from './helper';
const { test } = pwt;
test('foobar', async ({}) => {
ohMy();
});
`
}, {}, {
FORCE_COLOR: '0',
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Error: oh my');
expect(result.output).toContain(` at helper.ts:5`);
expect(result.output).toContain(` 4 | export function ohMy() {`);
expect(result.output).toContain(`> 5 | throw new Error('oh my');`);
expect(result.output).toContain(` | ^`);
}); });
test('should print slow tests', async ({ runInlineTest }) => { test('should print slow tests', async ({ runInlineTest }) => {