feat(test runner): implement test.describe.serial (#8132)

docs/src/test-api/class-test.md

@@ -245,6 +245,82 @@ A callback that is run immediately when calling [`method: Test.describe.only`].
 
 
 
+## method: Test.describe.serial
+
+Declares a group of tests that should always be run serially. If one of the tests fails, all subsequent tests are skipped. All tests in a group are retried together.
+
+:::note
+Using serial is not recommended. It is usually better to make your tests isolated, so they can be run independently.
+:::
+
+```js js-flavor=js
+test.describe.serial('group', () => {
+  test('runs first', async ({ page }) => {
+  });
+  test('runs second', async ({ page }) => {
+  });
+});
+```
+
+```js js-flavor=ts
+test.describe.serial('group', () => {
+  test('runs first', async ({ page }) => {
+  });
+  test('runs second', async ({ page }) => {
+  });
+});
+```
+
+### param: Test.describe.serial.title
+- `title` <[string]>
+
+Group title.
+
+### param: Test.describe.serial.callback
+- `callback` <[function]>
+
+A callback that is run immediately when calling [`method: Test.describe.serial`]. Any tests added in this callback will belong to the group.
+
+
+
+## method: Test.describe.serial.only
+
+Declares a focused group of tests that should always be run serially. If one of the tests fails, all subsequent tests are skipped. All tests in a group are retried together. If there are some focused tests or suites, all of them will be run but nothing else.
+
+:::note
+Using serial is not recommended. It is usually better to make your tests isolated, so they can be run independently.
+:::
+
+```js js-flavor=js
+test.describe.serial.only('group', () => {
+  test('runs first', async ({ page }) => {
+  });
+  test('runs second', async ({ page }) => {
+  });
+});
+```
+
+```js js-flavor=ts
+test.describe.serial.only('group', () => {
+  test('runs first', async ({ page }) => {
+  });
+  test('runs second', async ({ page }) => {
+  });
+});
+```
+
+### param: Test.describe.serial.only.title
+- `title` <[string]>
+
+Group title.
+
+### param: Test.describe.serial.only.callback
+- `callback` <[function]>
+
+A callback that is run immediately when calling [`method: Test.describe.serial.only`]. Any tests added in this callback will belong to the group.
+
+
+
 
 ## property: Test.expect
 - type: <[Object]>

src/test/dispatcher.ts

@@ -19,7 +19,7 @@ import path from 'path';
 import { EventEmitter } from 'events';
 import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc';
 import type { TestResult, Reporter, TestStep } from '../../types/testReporter';
-import { TestCase } from './test';
+import { Suite, TestCase } from './test';
 import { Loader } from './loader';
 
 export type TestGroup = {
@@ -125,7 +125,7 @@ export class Dispatcher {
       // When worker encounters error, we will stop it and create a new one.
       worker.stop();
 
-      const failedTestIds = new Set<string>();
+      const retryCandidates = new Set<string>();
 
       // In case of fatal error, report first remaining test as failing with this error,
       // and all others as skipped.
@@ -141,7 +141,7 @@ export class Dispatcher {
           result.error = params.fatalError;
           result.status = first ? 'failed' : 'skipped';
           this._reportTestEnd(test, result);
-          failedTestIds.add(test._id);
+          retryCandidates.add(test._id);
           first = false;
         }
         if (first) {
@@ -154,16 +154,50 @@ export class Dispatcher {
         // except for possible retries.
         remaining = [];
       }
-      if (params.failedTestId)
-        failedTestIds.add(params.failedTestId);
+
+      if (params.failedTestId) {
+        retryCandidates.add(params.failedTestId);
+
+        let outermostSerialSuite: Suite | undefined;
+        for (let parent = this._testById.get(params.failedTestId)!.test.parent; parent; parent = parent.parent) {
+          if (parent._serial)
+            outermostSerialSuite = parent;
+        }
+
+        if (outermostSerialSuite) {
+          // Failed test belongs to a serial suite. We should skip all future tests
+          // from the same serial suite.
+          remaining = remaining.filter(test => {
+            let parent = test.parent;
+            while (parent && parent !== outermostSerialSuite)
+              parent = parent.parent;
+
+            // Does not belong to the same serial suite, keep it.
+            if (!parent)
+              return true;
+
+            // Emulate a "skipped" run, and drop this test from remaining.
+            const { result } = this._testById.get(test._id)!;
+            this._reporter.onTestBegin?.(test, result);
+            result.status = 'skipped';
+            this._reportTestEnd(test, result);
+            return false;
+          });
+
+          // Add all tests from the same serial suite for possible retry.
+          // These will only be retried together, because they have the same
+          // "retries" setting and the same number of previous runs.
+          outermostSerialSuite.allTests().forEach(test => retryCandidates.add(test._id));
+        }
+      }
 
       // Only retry expected failures, not passes and only if the test failed.
-      for (const testId of failedTestIds) {
+      for (const testId of retryCandidates) {
         const pair = this._testById.get(testId)!;
         if (!this._isStopped && pair.test.expectedStatus === 'passed' && pair.test.results.length < pair.test.retries + 1) {
           pair.result = pair.test._appendTestResult();
           pair.steps = new Map();
-          remaining.unshift(pair.test);
+          remaining.push(pair.test);
         }
       }
 

src/test/reporters/base.ts

@@ -149,7 +149,7 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number
     const resultTokens = formatResultFailure(test, result, '    ');
     if (!resultTokens.length)
       continue;
-    const statusSuffix = result.status === 'passed' ? ' -- passed unexpectedly' : '';
+    const statusSuffix = (result.status === 'passed' && test.expectedStatus === 'failed') ? ' -- passed unexpectedly' : '';
     if (result.retry) {
       tokens.push('');
       tokens.push(colors.gray(pad(`    Retry #${result.retry}${statusSuffix}`, '-')));
@@ -185,7 +185,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase): string {
 
 function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string {
   const title = formatTestTitle(config, test);
-  const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : '';
+  const passedUnexpectedlySuffix = (test.results[0].status === 'passed' && test.expectedStatus === 'failed') ? ' -- passed unexpectedly' : '';
   const header = `${indent}${index ? index + ') ' : ''}${title}${passedUnexpectedlySuffix}`;
   return colors.red(pad(header, '='));
 }

src/test/test.ts

@@ -56,6 +56,7 @@ export class Suite extends Base implements reporterTypes.Suite {
   _timeout: number | undefined;
   _annotations: Annotations = [];
   _modifiers: Modifier[] = [];
+  _serial = false;
 
   _addTest(test: TestCase) {
     test.parent = this;
@@ -109,6 +110,7 @@ export class Suite extends Base implements reporterTypes.Suite {
     suite._annotations = this._annotations.slice();
     suite._modifiers = this._modifiers.slice();
     suite._isDescribe = this._isDescribe;
+    suite._serial = this._serial;
     return suite;
   }
 }
@@ -143,11 +145,12 @@ export class TestCase extends Base implements reporterTypes.TestCase {
   }
 
   outcome(): 'skipped' | 'expected' | 'unexpected' | 'flaky' {
-    if (!this.results.length || this.results[0].status === 'skipped')
+    const nonSkipped = this.results.filter(result => result.status !== 'skipped');
+    if (!nonSkipped.length)
       return 'skipped';
-    if (this.results.length === 1 && this.expectedStatus === this.results[0].status)
+    if (nonSkipped.every(result => result.status === this.expectedStatus))
       return 'expected';
-    if (this.results.some(result => result.status === this.expectedStatus))
+    if (nonSkipped.some(result => result.status === this.expectedStatus))
       return 'flaky';
     return 'unexpected';
   }

src/test/testType.ts

@@ -40,6 +40,8 @@ export class TestTypeImpl {
     test.only = wrapFunctionWithLocation(this._createTest.bind(this, 'only'));
     test.describe = wrapFunctionWithLocation(this._describe.bind(this, 'default'));
     test.describe.only = wrapFunctionWithLocation(this._describe.bind(this, 'only'));
+    test.describe.serial = wrapFunctionWithLocation(this._describe.bind(this, 'serial'));
+    test.describe.serial.only = wrapFunctionWithLocation(this._describe.bind(this, 'serial.only'));
     test.beforeEach = wrapFunctionWithLocation(this._hook.bind(this, 'beforeEach'));
     test.afterEach = wrapFunctionWithLocation(this._hook.bind(this, 'afterEach'));
     test.beforeAll = wrapFunctionWithLocation(this._hook.bind(this, 'beforeAll'));
@@ -75,7 +77,7 @@ export class TestTypeImpl {
       test.expectedStatus = 'skipped';
   }
 
-  private _describe(type: 'default' | 'only', location: Location, title: string, fn: Function) {
+  private _describe(type: 'default' | 'only' | 'serial' | 'serial.only', location: Location, title: string, fn: Function) {
     throwIfRunningInsideJest();
     const suite = currentlyLoadingFileSuite();
     if (!suite)
@@ -96,8 +98,10 @@ export class TestTypeImpl {
     child.location = location;
     suite._addSuite(child);
 
-    if (type === 'only')
+    if (type === 'only' || type === 'serial.only')
       child._only = true;
+    if (type === 'serial' || type === 'serial.only')
+      child._serial =  true;
 
     setCurrentlyLoadingFileSuite(child);
     fn();

tests/playwright-test/test-serial.spec.ts

@@ -0,0 +1,131 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('test.describe.serial should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = pwt;
+      test.describe.serial('serial suite', () => {
+        test('test1', async ({}) => {
+          console.log('\\n%%test1');
+        });
+        test('test2', async ({}) => {
+          console.log('\\n%%test2');
+        });
+
+        test.describe('inner suite', () => {
+          test('test3', async ({}) => {
+            console.log('\\n%%test3');
+            expect(1).toBe(2);
+          });
+          test('test4', async ({}) => {
+            console.log('\\n%%test4');
+          });
+        });
+
+        test('test5', async ({}) => {
+          console.log('\\n%%test5');
+        });
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(2);
+  expect(result.failed).toBe(1);
+  expect(result.skipped).toBe(2);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%test1',
+    '%%test2',
+    '%%test3',
+  ]);
+});
+
+test('test.describe.serial should work with retry', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = pwt;
+      test.describe.serial('serial suite', () => {
+        test('test1', async ({}) => {
+          console.log('\\n%%test1');
+        });
+        test('test2', async ({}) => {
+          console.log('\\n%%test2');
+        });
+
+        test.describe('inner suite', () => {
+          test('test3', async ({}, testInfo) => {
+            console.log('\\n%%test3');
+            expect(testInfo.retry).toBe(1);
+          });
+          test('test4', async ({}) => {
+            console.log('\\n%%test4');
+            expect(1).toBe(2);
+          });
+        });
+
+        test('test5', async ({}) => {
+          console.log('\\n%%test5');
+        });
+      });
+    `,
+  }, { retries: 1 });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(2);
+  expect(result.flaky).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.skipped).toBe(1);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%test1',
+    '%%test2',
+    '%%test3',
+    '%%test1',
+    '%%test2',
+    '%%test3',
+    '%%test4',
+  ]);
+});
+
+test('test.describe.serial.only should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async ({}) => {
+        console.log('\\n%%test1');
+      });
+      test.describe.serial.only('serial suite', () => {
+        test('test2', async ({}) => {
+          console.log('\\n%%test2');
+        });
+        test('test3', async ({}) => {
+          console.log('\\n%%test3');
+        });
+      });
+      test('test4', async ({}) => {
+        console.log('\\n%%test4');
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+  expect(result.failed).toBe(0);
+  expect(result.skipped).toBe(0);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%test2',
+    '%%test3',
+  ]);
+});

types/test.d.ts

@@ -1527,6 +1527,9 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    */
   describe: SuiteFunction & {
     only: SuiteFunction;
+    serial: SuiteFunction & {
+      only: SuiteFunction;
+    };
   };
   /**
    * Skips a test or a group of tests.

utils/generate_types/overrides-test.d.ts

@@ -223,6 +223,9 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
   only: TestFunction<TestArgs & WorkerArgs>;
   describe: SuiteFunction & {
     only: SuiteFunction;
+    serial: SuiteFunction & {
+      only: SuiteFunction;
+    };
   };
   skip(): void;
   skip(condition: boolean): void;