Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(test-runner): introduce test.step (#7956)

Browse source
This commit is contained in:
Pavel Feldman 2021-08-02 22:11:37 -07:00 committed by GitHub
parent 5803035c1b
commit 3b34e57ee4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 205 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
docs/src/test-api/class-test.md
View file

@ -709,6 +709,41 @@ Optional condition - either a boolean value, or a function that takes a fixtures
Optional description that will be reflected in a test report.
## method: Test.step
Declares a test step.
```js js-flavor=js
const { test, expect } = require('@playwright/test');
test('test', async ({ page }) => {
await test.step('Log in', async () => {
// ...
});
});
```
```js js-flavor=ts
import { test, expect } from '@playwright/test';
test('test', async ({ page }) => {
await test.step('Log in', async () => {
// ...
});
});
```
### param: Test.step.title
- `title` <[string]>
Step name.
### param: Test.step.body
- `body` <[function]\(\):[Promise]<[any]>>
Step body.
## method: Test.use

15
src/test/expect.ts
View file

@ -39,9 +39,10 @@ import {
toHaveValue
} from './matchers/matchers';
import { toMatchSnapshot } from './matchers/toMatchSnapshot';
import type { Expect, TestStatus } from './types';
import type { Expect, TestError } from './types';
import matchers from 'expect/build/matchers';
import { currentTestInfo } from './globals';
import { serializeError } from './util';
export const expect: Expect = expectLibrary as any;
expectLibrary.setState({ expand: false });
@ -78,22 +79,22 @@ function wrap(matcherName: string, matcher: any) {
const infix = this.isNot ? '.not' : '';
const completeStep = testInfo._addStep('expect', `expect${infix}.${matcherName}`);
const stack = new Error().stack;
const reportStepEnd = (result: any) => {
status = result.pass !== this.isNot ? 'passed' : 'failed';
let error: Error | undefined;
if (status === 'failed')
error = new Error(result.message());
const success = result.pass !== this.isNot;
let error: TestError | undefined;
if (!success)
error = { message: result.message(), stack };
completeStep?.(error);
return result;
};
const reportStepError = (error: Error) => {
completeStep?.(error);
completeStep?.(serializeError(error));
throw error;
};
let status: TestStatus = 'passed';
try {
const result = matcher.call(this, ...args);
if (result instanceof Promise)

19
src/test/testType.ts
View file

@ -19,7 +19,7 @@ import { currentlyLoadingFileSuite, currentTestInfo, setCurrentlyLoadingFileSuit
import { TestCase, Suite } from './test';
import { wrapFunctionWithLocation } from './transform';
import { Fixtures, FixturesWithLocation, Location, TestType } from './types';
import { errorWithLocation } from './util';
import { errorWithLocation, serializeError } from './util';
const countByFile = new Map<string, number>();
@ -49,6 +49,7 @@ export class TestTypeImpl {
test.fail = wrapFunctionWithLocation(this._modifier.bind(this, 'fail'));
test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow'));
test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this));
test.step = wrapFunctionWithLocation(this._step.bind(this));
test.use = wrapFunctionWithLocation(this._use.bind(this));
test.extend = wrapFunctionWithLocation(this._extend.bind(this));
test.declare = wrapFunctionWithLocation(this._declare.bind(this));
@ -146,7 +147,7 @@ export class TestTypeImpl {
const testInfo = currentTestInfo();
if (!testInfo)
throw errorWithLocation(location, `test.setTimeout() can only be called from a test file`);
throw errorWithLocation(location, `test.setTimeout() can only be called from a test`);
testInfo.setTimeout(timeout);
}
@ -157,6 +158,20 @@ export class TestTypeImpl {
suite._fixtureOverrides = { ...suite._fixtureOverrides, ...fixtures };
}
private async _step(location: Location, title: string, body: () => Promise<void>): Promise<void> {
const testInfo = currentTestInfo();
if (!testInfo)
throw errorWithLocation(location, `test.step() can only be called from a test`);
const complete = testInfo._addStep('test.step', title);
try {
await body();
complete();
} catch (e) {
complete(serializeError(e));
throw e;
}
}
private _extend(location: Location, fixtures: Fixtures) {
const fixturesWithLocation = { fixtures, location };
return new TestTypeImpl([...this.fixtures, fixturesWithLocation]).test;

2
src/test/types.ts
View file

@ -25,7 +25,7 @@ export type FixturesWithLocation = {
};
export type Annotations = { type: string, description?: string }[];
export type CompleteStepCallback = (error?: TestError) => void;
export type CompleteStepCallback = (error?: Error | TestError) => void;
export interface TestInfoImpl extends TestInfo {
_testFinished: Promise<void>;

4
src/test/workerRunner.ts
View file

@ -278,7 +278,9 @@ export class WorkerRunner extends EventEmitter {
wallTime: Date.now()
};
this.emit('stepBegin', payload);
return (error?: TestError) => {
return (error?: Error | TestError) => {
if (error instanceof Error)
error = serializeError(error);
const payload: StepEndPayload = {
testId,
stepId,

116
tests/playwright-test/reporter.spec.ts
View file

@ -168,6 +168,8 @@ test('should report expect steps', async ({ runInlineTest }) => {
}
onStepEnd(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
if (copy.error?.stack)
copy.error.stack = '<stack>';
console.log('%%%% end', JSON.stringify(copy));
}
}
@ -197,13 +199,13 @@ test('should report expect steps', async ({ runInlineTest }) => {
}, { reporter: '', workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
`%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
`%% begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`,
`%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{}}`,
`%% end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{\"message\":\"expect(received).toBeTruthy()\\n\\nReceived: false\",\"stack\":\"<stack>\"}}`,
`%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
@ -236,6 +238,8 @@ test('should report api steps', async ({ runInlineTest }) => {
}
onStepEnd(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
if (copy.error?.stack)
copy.error.stack = '<stack>';
console.log('%%%% end', JSON.stringify(copy));
}
}
@ -259,7 +263,7 @@ test('should report api steps', async ({ runInlineTest }) => {
}, { reporter: '', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
@ -274,3 +278,109 @@ test('should report api steps', async ({ runInlineTest }) => {
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
]);
});
test('should report api step failure', async ({ runInlineTest }) => {
const expectReporterJS = `
class Reporter {
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
console.log('%%%% begin', JSON.stringify(copy));
}
onStepEnd(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
if (copy.error?.stack)
copy.error.stack = '<stack>';
console.log('%%%% end', JSON.stringify(copy));
}
}
module.exports = Reporter;
`;
const result = await runInlineTest({
'reporter.ts': expectReporterJS,
'playwright.config.ts': `
module.exports = {
reporter: './reporter',
};
`,
'a.test.ts': `
const { test } = pwt;
test('fail', async ({ page }) => {
await page.setContent('<button></button>');
await page.click('input', { timeout: 1 });
});
`
}, { reporter: '', workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
`%% begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
`%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"page.click\",\"category\":\"pw:api\",\"error\":{\"message\":\"page.click: Timeout 1ms exceeded.\\n=========================== logs ===========================\\nwaiting for selector \\\"input\\\"\\n============================================================\",\"stack\":\"<stack>\"}}`,
`%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
]);
});
test('should report test.step', async ({ runInlineTest }) => {
const expectReporterJS = `
class Reporter {
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
console.log('%%%% begin', JSON.stringify(copy));
}
onStepEnd(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
if (copy.error?.stack)
copy.error.stack = '<stack>';
console.log('%%%% end', JSON.stringify(copy));
}
}
module.exports = Reporter;
`;
const result = await runInlineTest({
'reporter.ts': expectReporterJS,
'playwright.config.ts': `
module.exports = {
reporter: './reporter',
};
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async ({ page }) => {
await test.step('First step', async () => {
expect(1).toBe(2);
});
});
`
}, { reporter: '', workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
`%% begin {\"title\":\"First step\",\"category\":\"test.step\"}`,
`%% begin {\"title\":\"expect.toBe\",\"category\":\"expect\"}`,
`%% end {\"title\":\"expect.toBe\",\"category\":\"expect\",\"error\":{\"message\":\"expect(received).toBe(expected) // Object.is equality\\n\\nExpected: 2\\nReceived: 1\",\"stack\":\"<stack>\"}}`,
`%% end {\"title\":\"First step\",\"category\":\"test.step\",\"error\":{\"message\":\"expect(received).toBe(expected) // Object.is equality\\n\\nExpected: 2\\nReceived: 1\",\"stack\":\"<stack>\"}}`,
`%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
]);
});
function stripEscapedAscii(str: string) {
return str.replace(/\\u00[a-z0-9][a-z0-9]\[[^m]+m/g, '');
}

27
types/test.d.ts vendored
View file

@ -2093,6 +2093,33 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
* @param fixtures An object with fixture definitions.
*/
use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void;
/**
* Declares a test step.
*
* ```js js-flavor=js
* const { test, expect } = require('@playwright/test');
*
* test('test', async ({ page }) => {
* await test.step('Log in', async () => {
* // ...
* });
* });
* ```
*
* ```js js-flavor=ts
* import { test, expect } from '@playwright/test';
*
* test('test', async ({ page }) => {
* await test.step('Log in', async () => {
* // ...
* });
* });
* ```
*
* @param title Step name.
* @param body Step body.
*/
step(title: string, body: () => Promise<any>): Promise<any>;
/**
* `expect` function can be used to create test assertions. Read
* [expect library documentation](https://jestjs.io/docs/expect) for more details.

1
utils/generate_types/overrides-test.d.ts vendored
View file

@ -246,6 +246,7 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
beforeAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise<any> | any): void;
afterAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise<any> | any): void;
use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void;
step(title: string, body: () => Promise<any>): Promise<any>;
expect: Expect;
declare<T extends KeyValue = {}, W extends KeyValue = {}>(): TestType<TestArgs & T, WorkerArgs & W>;
extend<T, W extends KeyValue = {}>(fixtures: Fixtures<T, W, TestArgs, WorkerArgs>): TestType<TestArgs & T, WorkerArgs & W>;