feat(test): introduce fully parallel mode (#12446)

docs/src/test-api/class-testconfig.md

@@ -102,6 +102,14 @@ const config: PlaywrightTestConfig = {
 export default config;
 ```
 
+## property: TestConfig.fullyParallel
+- type: <[boolean]>
+
+Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same time.
+By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker process.
+
+You can configure entire test run to concurrently execute all tests in all files using this option.
+
 ## property: TestConfig.globalSetup
 - type: <[string]>
 

docs/src/test-api/class-testproject.md

@@ -116,6 +116,14 @@ Configuration for the `expect` assertion library.
 
 Use [`property: TestConfig.expect`] to change this option for all projects.
 
+## property: TestProject.fullyParallel
+- type: <[boolean]>
+
+Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same time.
+By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker process.
+
+You can configure entire test project to concurrently run all tests in all files using this option.
+
 ## property: TestProject.metadata
 - type: <[Object]>
 

docs/src/test-parallel-js.md

@@ -6,7 +6,8 @@ title: "Parallelism and sharding"
 Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same time.
 
 - By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker process.
-- Group tests with [`test.describe.parallel`](#parallelize-tests-in-a-single-file) to run **tests in a single file** in parallel.
+- Configure tests using [`test.describe.configure`](#parallelize-tests-in-a-single-file) to run **tests in a single file** in parallel.
+- You can configure entire project to have all tests in all files to run in parallel using [`property: TestProject.fullyParallel`] or [`property: TestConfig.fullyParallel`]
 - To **disable** parallelism limit the number of [workers to one](#disable-parallelism).
 
 You can control the number of [parallel worker processes](#limit-workers) and [limit the number of failures](#limit-failures-and-fail-fast) in the whole test suite for efficiency.

packages/playwright-test/src/cli.ts

@@ -40,6 +40,7 @@ export function addTestCommand(program: Command) {
   command.option('--debug', `Run tests with Playwright Inspector. Shortcut for "PWDEBUG=1" environment variable and "--timeout=0 --maxFailures=1 --headed --workers=1" options`);
   command.option('-c, --config <file>', `Configuration file, or a test directory with optional ${kDefaultConfigFiles.map(file => `"${file}"`).join('/')}`);
   command.option('--forbid-only', `Fail if test.only is called (default: false)`);
+  command.option('--fully-parallel', `Run all tests in parallel (default: false)`);
   command.option('-g, --grep <grep>', `Only run tests matching this regular expression (default: ".*")`);
   command.option('-gv, --grep-invert <grep>', `Only run tests that do not match this regular expression`);
   command.option('--global-timeout <timeout>', `Maximum time this test suite can run in milliseconds (default: unlimited)`);
@@ -196,6 +197,7 @@ function overridesFromOptions(options: { [key: string]: any }): Config {
   const shardPair = options.shard ? options.shard.split('/').map((t: string) => parseInt(t, 10)) : undefined;
   return {
     forbidOnly: options.forbidOnly ? true : undefined,
+    fullyParallel: options.fullyParallel ? true : undefined,
     globalTimeout: options.globalTimeout ? parseInt(options.globalTimeout, 10) : undefined,
     grep: options.grep ? forceRegExp(options.grep) : undefined,
     grepInvert: options.grepInvert ? forceRegExp(options.grepInvert) : undefined,

packages/playwright-test/src/loader.ts

@@ -93,6 +93,7 @@ export class Loader {
 
     this._fullConfig.rootDir = this._config.testDir || rootDir;
     this._fullConfig.forbidOnly = takeFirst(this._configOverrides.forbidOnly, this._config.forbidOnly, baseFullConfig.forbidOnly);
+    this._fullConfig.fullyParallel = takeFirst(this._configOverrides.fullyParallel, this._config.fullyParallel, baseFullConfig.fullyParallel);
     this._fullConfig.globalSetup = takeFirst(this._configOverrides.globalSetup, this._config.globalSetup, baseFullConfig.globalSetup);
     this._fullConfig.globalTeardown = takeFirst(this._configOverrides.globalTeardown, this._config.globalTeardown, baseFullConfig.globalTeardown);
     this._fullConfig.globalTimeout = takeFirst(this._configOverrides.globalTimeout, this._configOverrides.globalTimeout, this._config.globalTimeout, baseFullConfig.globalTimeout);
@@ -202,6 +203,7 @@ export class Loader {
     if (!path.isAbsolute(snapshotDir))
       snapshotDir = path.resolve(configDir, snapshotDir);
     const fullProject: FullProject = {
+      fullyParallel: takeFirst(this._configOverrides.fullyParallel, projectConfig.fullyParallel, this._config.fullyParallel, undefined),
       expect: takeFirst(this._configOverrides.expect, projectConfig.expect, this._config.expect, undefined),
       outputDir,
       repeatEach: takeFirst(this._configOverrides.repeatEach, projectConfig.repeatEach, this._config.repeatEach, 1),
@@ -429,6 +431,7 @@ function validateProject(file: string, project: Project, title: string) {
 
 const baseFullConfig: FullConfig = {
   forbidOnly: false,
+  fullyParallel: false,
   globalSetup: null,
   globalTeardown: null,
   globalTimeout: 0,

packages/playwright-test/src/runner.ts

@@ -286,6 +286,8 @@ export class Runner {
     for (const [project, files] of filesByProject) {
       const projectSuite = new Suite(project.config.name);
       projectSuite._projectConfig = project.config;
+      if (project.config.fullyParallel)
+        projectSuite._parallelMode = 'parallel';
       rootSuite._addSuite(projectSuite);
       for (const file of files) {
         const fileSuite = fileSuites.get(file);

packages/playwright-test/types/test.d.ts

@@ -125,6 +125,14 @@ interface TestProject {
    * all projects.
    */
   expect?: ExpectSettings;
+  /**
+   * Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same
+   * time. By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker
+   * process.
+   *
+   * You can configure entire test project to concurrently run all tests in all files using this option.
+   */
+  fullyParallel?: boolean;
   /**
    * Any JSON-serializable metadata that will be put directly to the test report.
    */
@@ -437,6 +445,14 @@ interface TestConfig {
    *
    */
   forbidOnly?: boolean;
+  /**
+   * Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same
+   * time. By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker
+   * process.
+   *
+   * You can configure entire test run to concurrently execute all tests in all files using this option.
+   */
+  fullyParallel?: boolean;
   /**
    * Path to the global setup file. This file will be required and run before all the tests. It must export a single function
    * that takes a [`TestConfig`] argument.
@@ -907,6 +923,14 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
    *
    */
   forbidOnly: boolean;
+  /**
+   * Playwright Test runs tests in parallel. In order to achieve that, it runs several worker processes that run at the same
+   * time. By default, **test files** are run in parallel. Tests in a single file are run in order, in the same worker
+   * process.
+   *
+   * You can configure entire test run to concurrently execute all tests in all files using this option.
+   */
+  fullyParallel: boolean;
   /**
    * Path to the global setup file. This file will be required and run before all the tests. It must export a single function
    * that takes a [`TestConfig`] argument.

tests/config/default.playwright.config.ts

@@ -49,6 +49,7 @@ const config: Config<CoverageWorkerOptions & PlaywrightWorkerOptions & Playwrigh
   timeout: video ? 60000 : 30000,
   globalTimeout: 5400000,
   workers: process.env.CI ? 1 : undefined,
+  fullyParallel: !process.env.CI,
   forbidOnly: !!process.env.CI,
   preserveOutput: process.env.CI ? 'failures-only' : 'always',
   retries: process.env.CI ? 3 : 0,

tests/playwright-test/test-parallel.spec.ts

@@ -116,3 +116,63 @@ test('test.describe.parallel should work in describe', async ({ runInlineTest })
   expect(result.output).toContain('%% worker=1');
   expect(result.output).toContain('%% worker=2');
 });
+
+test('config.fullyParallel should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { fullyParallel: true };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async ({}, testInfo) => {
+        console.log('\\n%% worker=' + testInfo.workerIndex);
+        await new Promise(f => setTimeout(f, 1000));
+      });
+      test('test2', async ({}, testInfo) => {
+        console.log('\\n%% worker=' + testInfo.workerIndex);
+        await new Promise(f => setTimeout(f, 1000));
+      });
+      test.describe('inner suite', () => {
+        test('test3', async ({}, testInfo) => {
+          console.log('\\n%% worker=' + testInfo.workerIndex);
+          await new Promise(f => setTimeout(f, 1000));
+        });
+      });
+    `,
+  }, { workers: 3 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  expect(result.output).toContain('%% worker=0');
+  expect(result.output).toContain('%% worker=1');
+  expect(result.output).toContain('%% worker=2');
+});
+
+test('project.fullyParallel should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [ { fullyParallel: true } ] };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async ({}, testInfo) => {
+        console.log('\\n%% worker=' + testInfo.workerIndex);
+        await new Promise(f => setTimeout(f, 1000));
+      });
+      test('test2', async ({}, testInfo) => {
+        console.log('\\n%% worker=' + testInfo.workerIndex);
+        await new Promise(f => setTimeout(f, 1000));
+      });
+      test.describe('inner suite', () => {
+        test('test3', async ({}, testInfo) => {
+          console.log('\\n%% worker=' + testInfo.workerIndex);
+          await new Promise(f => setTimeout(f, 1000));
+        });
+      });
+    `,
+  }, { workers: 3 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  expect(result.output).toContain('%% worker=0');
+  expect(result.output).toContain('%% worker=1');
+  expect(result.output).toContain('%% worker=2');
+});

utils/generate_types/overrides-test.d.ts

@@ -59,6 +59,7 @@ type ExpectSettings = {
 
 interface TestProject {
   expect?: ExpectSettings;
+  fullyParallel?: boolean;
   metadata?: any;
   name?: string;
   snapshotDir?: string;
@@ -117,6 +118,7 @@ type LiteralUnion<T extends U, U = string> = T | (U & { zz_IGNORE_ME?: never });
 
 interface TestConfig {
   forbidOnly?: boolean;
+  fullyParallel?: boolean;
   globalSetup?: string;
   globalTeardown?: string;
   globalTimeout?: number;
@@ -153,6 +155,7 @@ export interface Config<TestArgs = {}, WorkerArgs = {}> extends TestConfig {
 
 export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
   forbidOnly: boolean;
+  fullyParallel: boolean;
   globalSetup: string | null;
   globalTeardown: string | null;
   globalTimeout: number;