diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 7c8137e637..e3a1aeabce 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -245,6 +245,82 @@ A callback that is run immediately when calling [`method: Test.describe.only`]. +## method: Test.describe.serial + +Declares a group of tests that should always be run serially. If one of the tests fails, all subsequent tests are skipped. All tests in a group are retried together. + +:::note +Using serial is not recommended. It is usually better to make your tests isolated, so they can be run independently. +::: + +```js js-flavor=js +test.describe.serial('group', () => { + test('runs first', async ({ page }) => { + }); + test('runs second', async ({ page }) => { + }); +}); +``` + +```js js-flavor=ts +test.describe.serial('group', () => { + test('runs first', async ({ page }) => { + }); + test('runs second', async ({ page }) => { + }); +}); +``` + +### param: Test.describe.serial.title +- `title` <[string]> + +Group title. + +### param: Test.describe.serial.callback +- `callback` <[function]> + +A callback that is run immediately when calling [`method: Test.describe.serial`]. Any tests added in this callback will belong to the group. + + + +## method: Test.describe.serial.only + +Declares a focused group of tests that should always be run serially. If one of the tests fails, all subsequent tests are skipped. All tests in a group are retried together. If there are some focused tests or suites, all of them will be run but nothing else. + +:::note +Using serial is not recommended. It is usually better to make your tests isolated, so they can be run independently. +::: + +```js js-flavor=js +test.describe.serial.only('group', () => { + test('runs first', async ({ page }) => { + }); + test('runs second', async ({ page }) => { + }); +}); +``` + +```js js-flavor=ts +test.describe.serial.only('group', () => { + test('runs first', async ({ page }) => { + }); + test('runs second', async ({ page }) => { + }); +}); +``` + +### param: Test.describe.serial.only.title +- `title` <[string]> + +Group title. + +### param: Test.describe.serial.only.callback +- `callback` <[function]> + +A callback that is run immediately when calling [`method: Test.describe.serial.only`]. Any tests added in this callback will belong to the group. + + + ## property: Test.expect - type: <[Object]> diff --git a/src/test/dispatcher.ts b/src/test/dispatcher.ts index 3a4575717f..548ad5b398 100644 --- a/src/test/dispatcher.ts +++ b/src/test/dispatcher.ts @@ -19,7 +19,7 @@ import path from 'path'; import { EventEmitter } from 'events'; import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc'; import type { TestResult, Reporter, TestStep } from '../../types/testReporter'; -import { TestCase } from './test'; +import { Suite, TestCase } from './test'; import { Loader } from './loader'; export type TestGroup = { @@ -125,7 +125,7 @@ export class Dispatcher { // When worker encounters error, we will stop it and create a new one. worker.stop(); - const failedTestIds = new Set(); + const retryCandidates = new Set(); // In case of fatal error, report first remaining test as failing with this error, // and all others as skipped. @@ -141,7 +141,7 @@ export class Dispatcher { result.error = params.fatalError; result.status = first ? 'failed' : 'skipped'; this._reportTestEnd(test, result); - failedTestIds.add(test._id); + retryCandidates.add(test._id); first = false; } if (first) { @@ -154,16 +154,50 @@ export class Dispatcher { // except for possible retries. remaining = []; } - if (params.failedTestId) - failedTestIds.add(params.failedTestId); + + if (params.failedTestId) { + retryCandidates.add(params.failedTestId); + + let outermostSerialSuite: Suite | undefined; + for (let parent = this._testById.get(params.failedTestId)!.test.parent; parent; parent = parent.parent) { + if (parent._serial) + outermostSerialSuite = parent; + } + + if (outermostSerialSuite) { + // Failed test belongs to a serial suite. We should skip all future tests + // from the same serial suite. + remaining = remaining.filter(test => { + let parent = test.parent; + while (parent && parent !== outermostSerialSuite) + parent = parent.parent; + + // Does not belong to the same serial suite, keep it. + if (!parent) + return true; + + // Emulate a "skipped" run, and drop this test from remaining. + const { result } = this._testById.get(test._id)!; + this._reporter.onTestBegin?.(test, result); + result.status = 'skipped'; + this._reportTestEnd(test, result); + return false; + }); + + // Add all tests from the same serial suite for possible retry. + // These will only be retried together, because they have the same + // "retries" setting and the same number of previous runs. + outermostSerialSuite.allTests().forEach(test => retryCandidates.add(test._id)); + } + } // Only retry expected failures, not passes and only if the test failed. - for (const testId of failedTestIds) { + for (const testId of retryCandidates) { const pair = this._testById.get(testId)!; if (!this._isStopped && pair.test.expectedStatus === 'passed' && pair.test.results.length < pair.test.retries + 1) { pair.result = pair.test._appendTestResult(); pair.steps = new Map(); - remaining.unshift(pair.test); + remaining.push(pair.test); } } diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts index 4862c42f41..e54a424d1b 100644 --- a/src/test/reporters/base.ts +++ b/src/test/reporters/base.ts @@ -149,7 +149,7 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number const resultTokens = formatResultFailure(test, result, ' '); if (!resultTokens.length) continue; - const statusSuffix = result.status === 'passed' ? ' -- passed unexpectedly' : ''; + const statusSuffix = (result.status === 'passed' && test.expectedStatus === 'failed') ? ' -- passed unexpectedly' : ''; if (result.retry) { tokens.push(''); tokens.push(colors.gray(pad(` Retry #${result.retry}${statusSuffix}`, '-'))); @@ -185,7 +185,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase): string { function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string { const title = formatTestTitle(config, test); - const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : ''; + const passedUnexpectedlySuffix = (test.results[0].status === 'passed' && test.expectedStatus === 'failed') ? ' -- passed unexpectedly' : ''; const header = `${indent}${index ? index + ') ' : ''}${title}${passedUnexpectedlySuffix}`; return colors.red(pad(header, '=')); } diff --git a/src/test/test.ts b/src/test/test.ts index aeeacb37d9..5fc97ae651 100644 --- a/src/test/test.ts +++ b/src/test/test.ts @@ -56,6 +56,7 @@ export class Suite extends Base implements reporterTypes.Suite { _timeout: number | undefined; _annotations: Annotations = []; _modifiers: Modifier[] = []; + _serial = false; _addTest(test: TestCase) { test.parent = this; @@ -109,6 +110,7 @@ export class Suite extends Base implements reporterTypes.Suite { suite._annotations = this._annotations.slice(); suite._modifiers = this._modifiers.slice(); suite._isDescribe = this._isDescribe; + suite._serial = this._serial; return suite; } } @@ -143,11 +145,12 @@ export class TestCase extends Base implements reporterTypes.TestCase { } outcome(): 'skipped' | 'expected' | 'unexpected' | 'flaky' { - if (!this.results.length || this.results[0].status === 'skipped') + const nonSkipped = this.results.filter(result => result.status !== 'skipped'); + if (!nonSkipped.length) return 'skipped'; - if (this.results.length === 1 && this.expectedStatus === this.results[0].status) + if (nonSkipped.every(result => result.status === this.expectedStatus)) return 'expected'; - if (this.results.some(result => result.status === this.expectedStatus)) + if (nonSkipped.some(result => result.status === this.expectedStatus)) return 'flaky'; return 'unexpected'; } diff --git a/src/test/testType.ts b/src/test/testType.ts index acf5cbfc4b..875aa71949 100644 --- a/src/test/testType.ts +++ b/src/test/testType.ts @@ -40,6 +40,8 @@ export class TestTypeImpl { test.only = wrapFunctionWithLocation(this._createTest.bind(this, 'only')); test.describe = wrapFunctionWithLocation(this._describe.bind(this, 'default')); test.describe.only = wrapFunctionWithLocation(this._describe.bind(this, 'only')); + test.describe.serial = wrapFunctionWithLocation(this._describe.bind(this, 'serial')); + test.describe.serial.only = wrapFunctionWithLocation(this._describe.bind(this, 'serial.only')); test.beforeEach = wrapFunctionWithLocation(this._hook.bind(this, 'beforeEach')); test.afterEach = wrapFunctionWithLocation(this._hook.bind(this, 'afterEach')); test.beforeAll = wrapFunctionWithLocation(this._hook.bind(this, 'beforeAll')); @@ -75,7 +77,7 @@ export class TestTypeImpl { test.expectedStatus = 'skipped'; } - private _describe(type: 'default' | 'only', location: Location, title: string, fn: Function) { + private _describe(type: 'default' | 'only' | 'serial' | 'serial.only', location: Location, title: string, fn: Function) { throwIfRunningInsideJest(); const suite = currentlyLoadingFileSuite(); if (!suite) @@ -96,8 +98,10 @@ export class TestTypeImpl { child.location = location; suite._addSuite(child); - if (type === 'only') + if (type === 'only' || type === 'serial.only') child._only = true; + if (type === 'serial' || type === 'serial.only') + child._serial = true; setCurrentlyLoadingFileSuite(child); fn(); diff --git a/tests/playwright-test/test-serial.spec.ts b/tests/playwright-test/test-serial.spec.ts new file mode 100644 index 0000000000..fec4824925 --- /dev/null +++ b/tests/playwright-test/test-serial.spec.ts @@ -0,0 +1,131 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('test.describe.serial should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test.describe.serial('serial suite', () => { + test('test1', async ({}) => { + console.log('\\n%%test1'); + }); + test('test2', async ({}) => { + console.log('\\n%%test2'); + }); + + test.describe('inner suite', () => { + test('test3', async ({}) => { + console.log('\\n%%test3'); + expect(1).toBe(2); + }); + test('test4', async ({}) => { + console.log('\\n%%test4'); + }); + }); + + test('test5', async ({}) => { + console.log('\\n%%test5'); + }); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(2); + expect(result.failed).toBe(1); + expect(result.skipped).toBe(2); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%test1', + '%%test2', + '%%test3', + ]); +}); + +test('test.describe.serial should work with retry', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test.describe.serial('serial suite', () => { + test('test1', async ({}) => { + console.log('\\n%%test1'); + }); + test('test2', async ({}) => { + console.log('\\n%%test2'); + }); + + test.describe('inner suite', () => { + test('test3', async ({}, testInfo) => { + console.log('\\n%%test3'); + expect(testInfo.retry).toBe(1); + }); + test('test4', async ({}) => { + console.log('\\n%%test4'); + expect(1).toBe(2); + }); + }); + + test('test5', async ({}) => { + console.log('\\n%%test5'); + }); + }); + `, + }, { retries: 1 }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(2); + expect(result.flaky).toBe(1); + expect(result.failed).toBe(1); + expect(result.skipped).toBe(1); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%test1', + '%%test2', + '%%test3', + '%%test1', + '%%test2', + '%%test3', + '%%test4', + ]); +}); + +test('test.describe.serial.only should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test('test1', async ({}) => { + console.log('\\n%%test1'); + }); + test.describe.serial.only('serial suite', () => { + test('test2', async ({}) => { + console.log('\\n%%test2'); + }); + test('test3', async ({}) => { + console.log('\\n%%test3'); + }); + }); + test('test4', async ({}) => { + console.log('\\n%%test4'); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(0); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%test2', + '%%test3', + ]); +}); diff --git a/types/test.d.ts b/types/test.d.ts index 82eb5dce63..1c2fcd57d2 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -1527,6 +1527,9 @@ export interface TestType; describe: SuiteFunction & { only: SuiteFunction; + serial: SuiteFunction & { + only: SuiteFunction; + }; }; skip(): void; skip(condition: boolean): void;