Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(test): step.fail and step.fixme modifiers

Browse source 
Reference https://github.com/microsoft/playwright/issues/10033
This commit is contained in:
Yury Semikhatsky 2024-12-13 17:24:23 -08:00
parent b58a4762f4
commit 4a62306040
7 changed files with 517 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
docs/src/test-api/class-test.md
View file

@ -1773,6 +1773,114 @@ Specifies a custom location for the step to be shown in test reports and trace v
Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
## async method: Test.step.fail
* since: v1.50
- returns: <[void]>
Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed.
To declare a "failing" step:
* `test.step.fail(title, body)`
**Usage**
You can declare a test step as failing, so that Playwright ensures it actually fails.
```js
import { test, expect } from '@playwright/test';
test('my test', async ({ page }) => {
// ...
await test.step.fail('not yet ready', async () => {
// ...
});
});
```
### param: Test.step.fail.title
* since: v1.50
- `title` <[string]>
Step name.
### param: Test.step.fail.body
* since: v1.50
- `body` <[function]\(\):[Promise]<[any]>>
Step body.
### option: Test.step.fail.box
* since: v1.50
- `box` <boolean>
Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details.
### option: Test.step.fail.location
* since: v1.50
- `location` <[Location]>
Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
### option: Test.step.fail.timeout
* since: v1.50
- `timeout` <[float]>
Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
## async method: Test.step.fixme
* since: v1.50
- returns: <[void]>
Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step.
To declare a "fixme" step:
* `test.step.fixme(title, body)`
**Usage**
You can declare a test step as failing, so that Playwright ensures it actually fails.
```js
import { test, expect } from '@playwright/test';
test('my test', async ({ page }) => {
// ...
await test.step.fixme('not yet ready', async () => {
// ...
});
});
```
### param: Test.step.fixme.title
* since: v1.50
- `title` <[string]>
Step name.
### param: Test.step.fixme.body
* since: v1.50
- `body` <[function]\(\):[Promise]<[any]>>
Step body.
### option: Test.step.fixme.box
* since: v1.50
- `box` <boolean>
Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details.
### option: Test.step.fixme.location
* since: v1.50
- `location` <[Location]>
Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
### option: Test.step.fixme.timeout
* since: v1.50
- `timeout` <[float]>
Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
## method: Test.use
* since: v1.10

37
packages/playwright/src/common/testType.ts
View file

@ -56,7 +56,10 @@ export class TestTypeImpl {
test.fail.only = wrapFunctionWithLocation(this._createTest.bind(this, 'fail.only'));
test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow'));
test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this));
test.step = this._step.bind(this);
test.step = this._step.bind(this, 'pass');
test.step.skip = this._step.bind(this, 'skip');
test.step.fail = this._step.bind(this, 'fail');
test.step.fixme = this._step.bind(this, 'fixme');
test.use = wrapFunctionWithLocation(this._use.bind(this));
test.extend = wrapFunctionWithLocation(this._extend.bind(this));
test.info = () => {
@ -257,22 +260,40 @@ export class TestTypeImpl {
suite._use.push({ fixtures, location });
}
async _step<T>(title: string, body: () => T | Promise<T>, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise<T> {
async _step<T>(expectation: 'pass'|'fail'|'fixme'|'skip', title: string, body: () => T | Promise<T>, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise<T> {
const testInfo = currentTestInfo();
if (!testInfo)
throw new Error(`test.step() can only be called from a test`);
if (expectation === 'skip' || expectation === 'fixme')
return undefined as T;
const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box });
return await zones.run('stepZone', step, async () => {
let result;
let error;
try {
const result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0);
if (result.timedOut)
throw new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`);
step.complete({});
return result.result;
} catch (error) {
result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0);
} catch (e) {
error = e;
}
if (result?.timedOut) {
const error = new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`);
step.complete({ error });
throw error;
}
const expectedToFail = expectation === 'fail';
if (error) {
step.complete({ error });
if (expectedToFail)
return undefined as T;
throw error;
}
if (expectedToFail) {
error = new Error(`Step is expected to fail, but passed`);
step.complete({ error });
throw error;
}
step.complete({});
return result!.result;
});
}

215
packages/playwright/types/test.d.ts vendored
View file

@ -5551,7 +5551,220 @@ export interface TestType<TestArgs extends {}, WorkerArgs extends {}> {
* @param body Step body.
* @param options
*/
step<T>(title: string, body: () => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
step: {
/**
* Declares a test step that is shown in the report.
*
* **Usage**
*
* ```js
* import { test, expect } from '@playwright/test';
*
* test('test', async ({ page }) => {
* await test.step('Log in', async () => {
* // ...
* });
*
* await test.step('Outer step', async () => {
* // ...
* // You can nest steps inside each other.
* await test.step('Inner step', async () => {
* // ...
* });
* });
* });
* ```
*
* **Details**
*
* The method returns the value returned by the step callback.
*
* ```js
* import { test, expect } from '@playwright/test';
*
* test('test', async ({ page }) => {
* const user = await test.step('Log in', async () => {
* // ...
* return 'john';
* });
* expect(user).toBe('john');
* });
* ```
*
* **Decorator**
*
* You can use TypeScript method decorators to turn a method into a step. Each call to the decorated method will show
* up as a step in the report.
*
* ```js
* function step(target: Function, context: ClassMethodDecoratorContext) {
* return function replacementMethod(...args: any) {
* const name = this.constructor.name + '.' + (context.name as string);
* return test.step(name, async () => {
* return await target.call(this, ...args);
* });
* };
* }
*
* class LoginPage {
* constructor(readonly page: Page) {}
*
* @step
* async login() {
* const account = { username: 'Alice', password: 's3cr3t' };
* await this.page.getByLabel('Username or email address').fill(account.username);
* await this.page.getByLabel('Password').fill(account.password);
* await this.page.getByRole('button', { name: 'Sign in' }).click();
* await expect(this.page.getByRole('button', { name: 'View profile and more' })).toBeVisible();
* }
* }
*
* test('example', async ({ page }) => {
* const loginPage = new LoginPage(page);
* await loginPage.login();
* });
* ```
*
* **Boxing**
*
* When something inside a step fails, you would usually see the error pointing to the exact action that failed. For
* example, consider the following login step:
*
* ```js
* async function login(page) {
* await test.step('login', async () => {
* const account = { username: 'Alice', password: 's3cr3t' };
* await page.getByLabel('Username or email address').fill(account.username);
* await page.getByLabel('Password').fill(account.password);
* await page.getByRole('button', { name: 'Sign in' }).click();
* await expect(page.getByRole('button', { name: 'View profile and more' })).toBeVisible();
* });
* }
*
* test('example', async ({ page }) => {
* await page.goto('https://github.com/login');
* await login(page);
* });
* ```
*
* ```txt
* Error: Timed out 5000ms waiting for expect(locator).toBeVisible()
* ... error details omitted ...
*
* 8 | await page.getByRole('button', { name: 'Sign in' }).click();
* > 9 | await expect(page.getByRole('button', { name: 'View profile and more' })).toBeVisible();
* | ^
* 10 | });
* ```
*
* As we see above, the test may fail with an error pointing inside the step. If you would like the error to highlight
* the "login" step instead of its internals, use the `box` option. An error inside a boxed step points to the step
* call site.
*
* ```js
* async function login(page) {
* await test.step('login', async () => {
* // ...
* }, { box: true }); // Note the "box" option here.
* }
* ```
*
* ```txt
* Error: Timed out 5000ms waiting for expect(locator).toBeVisible()
* ... error details omitted ...
*
* 14 | await page.goto('https://github.com/login');
* > 15 | await login(page);
* | ^
* 16 | });
* ```
*
* You can also create a TypeScript decorator for a boxed step, similar to a regular step decorator above:
*
* ```js
* function boxedStep(target: Function, context: ClassMethodDecoratorContext) {
* return function replacementMethod(...args: any) {
* const name = this.constructor.name + '.' + (context.name as string);
* return test.step(name, async () => {
* return await target.call(this, ...args);
* }, { box: true }); // Note the "box" option here.
* };
* }
*
* class LoginPage {
* constructor(readonly page: Page) {}
*
* @boxedStep
* async login() {
* // ....
* }
* }
*
* test('example', async ({ page }) => {
* const loginPage = new LoginPage(page);
* await loginPage.login(); // <-- Error will be reported on this line.
* });
* ```
*
* @param title Step name.
* @param body Step body.
* @param options
*/
<T>(title: string, body: () => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
/**
* Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step.
*
* To declare a "fixme" step:
* - `test.step.fixme(title, body)`
*
* **Usage**
*
* You can declare a test step as failing, so that Playwright ensures it actually fails.
*
* ```js
* import { test, expect } from '@playwright/test';
*
* test('my test', async ({ page }) => {
* // ...
* await test.step.fixme('not yet ready', async () => {
* // ...
* });
* });
* ```
*
* @param title Step name.
* @param body Step body.
* @param options
*/
fixme(title: string, body: () => void | Promise<void>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
/**
* Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is
* useful for documentation purposes to acknowledge that some functionality is broken until it is fixed.
*
* To declare a "failing" step:
* - `test.step.fail(title, body)`
*
* **Usage**
*
* You can declare a test step as failing, so that Playwright ensures it actually fails.
*
* ```js
* import { test, expect } from '@playwright/test';
*
* test('my test', async ({ page }) => {
* // ...
* await test.step.fail('not yet ready', async () => {
* // ...
* });
* });
* ```
*
* @param title Step name.
* @param body Step body.
* @param options
*/
fail(title: string, body: () => void | Promise<void>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
}
/**
* `expect` function can be used to create test assertions. Read more about [test assertions](https://playwright.dev/docs/test-assertions).
*

42
tests/page/page-check.spec.ts
View file

@ -17,16 +17,42 @@
import { test as it, expect } from './pageTest';
it('should check the box @smoke', async ({ page }) => {
await page.setContent(`<input id='checkbox' type='checkbox'></input>`);
await page.check('input');
expect(await page.evaluate(() => window['checkbox'].checked)).toBe(true);
});
it('should not check the checked box', async ({ page }) => {
await page.setContent(`<input id='checkbox' type='checkbox' checked></input>`);
await page.check('input');
expect(await page.evaluate(() => window['checkbox'].checked)).toBe(true);
await it.step('outer step 1', async () => {
await it.step.fail('inner step 1.1', async () => {
throw new Error('inner step 1.1 failed');
});
await it.step.fixme('inner step 1.2', async () => {
console.log('inner step 1.2 skipped');
});
});
await it.step('outer step 2', async () => {
await it.step.fixme('inner step 2.1', async () => {
console.log('inner step 2.1 expected to fail');
});
await it.step('inner step 2.2', async () => {
expect(1).toBe(1);
});
});
await it.step.fail('outer step 3', async () => {
throw new Error('outer step 3 failed');
});
});
it.fail('exp falure', async ({ page }) => {
await page.setContent(`<input id='checkbox' type='checkbox' checked></input>`);
expect(1).toBe(2);
});
it.skip('exp skipped', async ({ page }) => {
await page.setContent(`<input id='checkbox' type='checkbox' checked></input>`);
expect(1).toBe(2);
});
it.fixme('exp fixme', async ({ page }) => {
await page.setContent(`<input id='checkbox' type='checkbox' checked></input>`);
expect(1).toBe(2);
});
it('should uncheck the box', async ({ page }) => {

100
tests/playwright-test/test-step.spec.ts
View file

@ -1494,3 +1494,103 @@ fixture | fixture: context
`);
});
test('test.step.fail and test.step.fixme should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': stepIndentReporter,
'playwright.config.ts': `module.exports = { reporter: './reporter' };`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({ }) => {
await test.step('outer step 1', async () => {
await test.step.fail('inner step 1.1', async () => {
throw new Error('inner step 1.1 failed');
});
await test.step.fixme('inner step 1.2', async () => {});
await test.step('inner step 1.3', async () => {});
});
await test.step('outer step 2', async () => {
await test.step.fixme('inner step 2.1', async () => {});
await test.step('inner step 2.2', async () => {
expect(1).toBe(1);
});
});
await test.step.fail('outer step 3', async () => {
throw new Error('outer step 3 failed');
});
});
`
}, { reporter: '', workers: 1, timeout: 3000 });
expect(result.exitCode).toBe(0);
expect(result.report.stats.expected).toBe(1);
expect(result.report.stats.unexpected).toBe(0);
expect(stripAnsi(result.output)).toBe(`
hook |Before Hooks
test.step |outer step 1 @ a.test.ts:4
test.step | inner step 1.1 @ a.test.ts:5
test.step | ↪ error: Error: inner step 1.1 failed
test.step | inner step 1.3 @ a.test.ts:9
test.step |outer step 2 @ a.test.ts:11
test.step | inner step 2.2 @ a.test.ts:13
expect | expect.toBe @ a.test.ts:14
test.step |outer step 3 @ a.test.ts:17
test.step |↪ error: Error: outer step 3 failed
hook |After Hooks
`);
});
test('timeout inside test.step.fail is an error', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': stepIndentReporter,
'playwright.config.ts': `module.exports = { reporter: './reporter' };`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('test 2', async ({ }) => {
await test.step('outer step 2', async () => {
await test.step.fail('inner step 2', async () => {
await new Promise(() => {});
});
});
});
`
}, { reporter: '', workers: 2, timeout: 1000 });
expect(result.exitCode).toBe(1);
expect(result.report.stats.unexpected).toBe(1);
expect(stripAnsi(result.output)).toBe(`
hook |Before Hooks
test.step |outer step 2 @ a.test.ts:4
test.step | inner step 2 @ a.test.ts:5
hook |After Hooks
hook |Worker Cleanup
|Test timeout of 1000ms exceeded.
`);
});
test('skip test.step.fixme body', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': stepIndentReporter,
'playwright.config.ts': `module.exports = { reporter: './reporter' };`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({ }) => {
let didRun = false;
await test.step('outer step 2', async () => {
await test.step.fixme('inner step 2', async () => {
didRun = true;
});
});
expect(didRun).toBe(false);
});
`
}, { reporter: '', workers: 1, timeout: 1000 });
expect(result.exitCode).toBe(0);
expect(result.report.stats.expected).toBe(1);
expect(stripAnsi(result.output)).toBe(`
hook |Before Hooks
test.step |outer step 2 @ a.test.ts:5
expect |expect.toBe @ a.test.ts:10
hook |After Hooks
`);
});

27
tests/playwright-test/types-2.spec.ts
View file

@ -204,3 +204,30 @@ test('step should inherit return type from its callback ', async ({ runTSC }) =>
});
expect(result.exitCode).toBe(0);
});
test('step.fail and step.fixme return void ', async ({ runTSC }) => {
const result = await runTSC({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('test step.fail', async ({ }) => {
// @ts-expect-error
const bad1: string = await test.step.fail('my step', () => { });
// @ts-expect-error
const bad2: void = await test.step.fail('my step', async () => {
return 10;
});
const good: void = await test.step.fail('my step', async () => { });
});
test('test step.fixme', async ({ }) => {
// @ts-expect-error
const bad1: string = await test.step.fixme('my step', () => { });
// @ts-expect-error
const bad2: void = await test.step.fixme('my step', async () => {
return 10;
});
const good: void = await test.step.fixme('my step', async () => { });
});
`
});
expect(result.exitCode).toBe(0);
});

6
utils/generate_types/overrides-test.d.ts vendored
View file

@ -162,7 +162,11 @@ export interface TestType<TestArgs extends {}, WorkerArgs extends {}> {
afterAll(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<any> | any): void;
afterAll(title: string, inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<any> | any): void;
use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void;
step<T>(title: string, body: () => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
step: {
<T>(title: string, body: () => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
fixme(title: string, body: () => void | Promise<void>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
fail(title: string, body: () => void | Promise<void>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
}
expect: Expect<{}>;
extend<T extends {}, W extends {} = {}>(fixtures: Fixtures<T, W, TestArgs, WorkerArgs>): TestType<TestArgs & T, WorkerArgs & W>;
info(): TestInfo;