feat(runner): project execution schedule (#17548)

docs/src/test-api/class-testconfig.md

@@ -232,6 +232,13 @@ Filter to only run tests with a title **not** matching one of the patterns. This
 
 `grepInvert` option is also useful for [tagging tests](../test-annotations.md#tag-tests).
 
+## property: TestConfig.groups
+* since: v1.27
+- type: ?<[Object]<[string],[Array]<[string]|[Array]<[string]|[Object]>>>>
+  - `project` <[string]|[Array]<[string]>> Project name(s).
+
+Project groups that control project execution order.
+
 ## property: TestConfig.ignoreSnapshots
 * since: v1.26
 - type: ?<[boolean]>

packages/playwright-test/src/cli.ts

@@ -58,6 +58,7 @@ function addTestCommand(program: Command) {
   command.option('--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`);
   command.option('--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`);
   command.option('--project <project-name...>', `Only run tests from the specified list of projects (default: run all projects)`);
+  command.option('--group <project-group-name>', `Only run tests from the specified project group (default: run all projects from the 'default' group or just all projects if 'default' group is not defined).`);
   command.option('--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`);
   command.option('--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`);
   command.option('-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`);
@@ -166,6 +167,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
     listOnly: !!opts.list,
     testFileFilters,
     projectFilter: opts.project || undefined,
+    projectGroup: opts.group,
     watchMode: !!process.env.PW_TEST_WATCH,
     passWithNoTests: opts.passWithNoTests,
   });

packages/playwright-test/src/loader.ts

@@ -27,7 +27,7 @@ import * as os from 'os';
 import type { BuiltInReporter, ConfigCLIOverrides } from './runner';
 import type { Reporter } from '../types/testReporter';
 import { builtInReporters } from './runner';
-import { isRegExp, calculateSha1 } from 'playwright-core/lib/utils';
+import { isRegExp, calculateSha1, isString, isObject } from 'playwright-core/lib/utils';
 import { serializeError } from './util';
 import { FixturePool, isFixtureOption } from './fixtures';
 import type { TestTypeImpl } from './testType';
@@ -167,6 +167,7 @@ export class Loader {
     this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata);
     this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath));
     this._assignUniqueProjectIds(this._fullConfig.projects);
+    this._fullConfig.groups = config.groups;
   }
 
   private _assignUniqueProjectIds(projects: FullProjectInternal[]) {
@@ -538,6 +539,8 @@ function validateConfig(file: string, config: Config) {
     });
   }
 
+  validateProjectGroups(file, config);
+
   if ('quiet' in config && config.quiet !== undefined) {
     if (typeof config.quiet !== 'boolean')
       throw errorWithFile(file, `config.quiet must be a boolean`);
@@ -644,6 +647,54 @@ function validateProject(file: string, project: Project, title: string) {
   }
 }
 
+function validateProjectGroups(file: string, config: Config) {
+  if (config.groups === undefined)
+    return;
+  const projectNames = new Set(config.projects?.filter(p => !!p.name).map(p => p.name));
+  for (const [groupName, group] of Object.entries(config.groups)) {
+    function validateProjectReference(projectName: string) {
+      if (projectName.trim() === '')
+        throw errorWithFile(file, `config.groups.${groupName} refers to an empty project name`);
+      if (!projectNames.has(projectName))
+        throw errorWithFile(file, `config.groups.${groupName} refers to an unknown project '${projectName}'`);
+    }
+    for (const step of group) {
+      if (isString(step)) {
+        validateProjectReference(step);
+      } else if (Array.isArray(step)) {
+        const parallelProjectNames = new Set();
+        for (const item of step) {
+          let projectName;
+          if (isString(item)) {
+            validateProjectReference(item);
+            projectName = item;
+          } else if (isObject(item)) {
+            const project = (item as any).project;
+            if (isString(project)) {
+              validateProjectReference(project);
+            } else if (Array.isArray(project)) {
+              project.forEach(name => {
+                if (!isString(name))
+                  throw errorWithFile(file, `config.groups.${groupName}[*].project contains non string value.`);
+                validateProjectReference(name);
+              });
+            }
+            projectName = project;
+          } else {
+            throw errorWithFile(file, `config.groups.${groupName} unexpected group entry ${JSON.stringify(step, null, 2)}`);
+          }
+          // We can relax this later.
+          if (parallelProjectNames.has(projectName))
+            throw errorWithFile(file, `config.groups.${groupName} group mentions project '${projectName}' twice in one parallel group`);
+          parallelProjectNames.add(projectName);
+        }
+      } else {
+        throw errorWithFile(file, `config.groups.${groupName} unexpected group entry ${JSON.stringify(step, null, 2)}`);
+      }
+    }
+  }
+}
+
 export const baseFullConfig: FullConfigInternal = {
   forbidOnly: false,
   fullyParallel: false,
@@ -670,7 +721,7 @@ export const baseFullConfig: FullConfigInternal = {
   _webServers: [],
   _globalOutputDir: path.resolve(process.cwd()),
   _configDir: '',
-  _testGroupsCount: 0,
+  _maxConcurrentTestGroups: 0,
   _ignoreSnapshots: false,
   _workerIsolation: 'isolate-pools',
 };

packages/playwright-test/src/reporters/base.ts

@@ -121,7 +121,7 @@ export class BaseReporter implements ReporterInternal  {
   }
 
   protected generateStartingMessage() {
-    const jobs = Math.min(this.config.workers, this.config._testGroupsCount);
+    const jobs = Math.min(this.config.workers, this.config._maxConcurrentTestGroups);
     const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
     if (this.config._watchMode)
       return `\nRunning tests in the --watch mode`;

packages/playwright-test/src/runner.ts

@@ -47,16 +47,24 @@ import { setRunnerToAddPluginsTo } from './plugins';
 import { webServerPluginsForConfig } from './plugins/webServerPlugin';
 import { dockerPlugin } from './plugins/dockerPlugin';
 import { MultiMap } from 'playwright-core/lib/utils/multimap';
+import { isString, assert } from 'playwright-core/lib/utils';
 
 const removeFolderAsync = promisify(rimraf);
 const readDirAsync = promisify(fs.readdir);
 const readFileAsync = promisify(fs.readFile);
 export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs'];
 
+// Project group is a sequence of run phases.
+type RunPhase = {
+  testFileFilters: TestFileFilter[];
+  projectFilter?: string[];
+};
+
 type RunOptions = {
   listOnly?: boolean;
   testFileFilters?: TestFileFilter[];
   projectFilter?: string[];
+  projectGroup?: string;
   watchMode?: boolean;
   passWithNoTests?: boolean;
 };
@@ -227,6 +235,58 @@ export class Runner {
     return report;
   }
 
+
+  private _collectRunPhases(options: RunOptions) {
+    const config = this._loader.fullConfig();
+
+    let projectGroup = options.projectGroup;
+    if (options.projectFilter) {
+      if (projectGroup)
+        throw new Error('--group option can not be combined with --project');
+    } else {
+      if (!projectGroup && config.groups?.default && !options.testFileFilters?.length)
+        projectGroup = 'default';
+      if (projectGroup) {
+        if (config.shard)
+          throw new Error(`Project group '${projectGroup}' cannot be combined with --shard`);
+      }
+    }
+
+    const phases: RunPhase[] = [];
+    if (projectGroup) {
+      const group = config.groups?.[projectGroup];
+      if (!group)
+        throw new Error(`Cannot find project group '${projectGroup}' in the config`);
+      for (const entry of group) {
+        const projectFilter: string[] = [];
+        const testFileFilters: TestFileFilter[] = [];
+        if (isString(entry)) {
+          projectFilter.push(entry);
+        } else {
+          for (const p of entry) {
+            if (isString(p))
+              projectFilter.push(p);
+            else if (isString(p.project))
+              projectFilter.push(p.project);
+            else
+              projectFilter.push(...p.project);
+          }
+        }
+        // TODO: filter per project set.
+        phases.push({
+          testFileFilters,
+          projectFilter
+        });
+      }
+    } else {
+      phases.push({
+        projectFilter: options.projectFilter,
+        testFileFilters: options.testFileFilters || [],
+      });
+    }
+    return phases;
+  }
+
   private async _collectFiles(testFileFilters: TestFileFilter[], projectNames?: string[]): Promise<Map<FullProjectInternal, string[]>> {
     const testFileFilter = testFileFilters.length ? createFileMatcher(testFileFilters.map(e => e.re || e.exact || '')) : () => true;
     let projectsToFind: Set<string> | undefined;
@@ -270,74 +330,86 @@ export class Runner {
   }
 
   private async _run(options: RunOptions): Promise<FullResult> {
-    const testFileFilters = options.testFileFilters || [];
-    const filesByProject = await this._collectFiles(testFileFilters, options.projectFilter);
-
-    const allTestFiles = new Set<string>();
-    for (const files of filesByProject.values())
-      files.forEach(file => allTestFiles.add(file));
-
     const config = this._loader.fullConfig();
-
     const fatalErrors: TestError[] = [];
-
+    // Each entry is an array of test groups that can be run concurrently. All
-    // 1. Add all tests.
+    // test groups from the previos entries must finish before entry starts.
-    const preprocessRoot = new Suite('', 'root');
+    const concurrentTestGroups = [];
-    for (const file of allTestFiles) {
-      const fileSuite = await this._loader.loadTestFile(file, 'runner');
-      if (fileSuite._loadError)
-        fatalErrors.push(fileSuite._loadError);
-      preprocessRoot._addSuite(fileSuite);
-    }
-
-    // 2. Complain about duplicate titles.
-    const duplicateTitlesError = createDuplicateTitlesError(config, preprocessRoot);
-    if (duplicateTitlesError)
-      fatalErrors.push(duplicateTitlesError);
-
-    // 3. Filter tests to respect line/column filter.
-    filterByFocusedLine(preprocessRoot, testFileFilters);
-
-    // 4. Complain about only.
-    if (config.forbidOnly) {
-      const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
-      if (onlyTestsAndSuites.length > 0)
-        fatalErrors.push(createForbidOnlyError(config, onlyTestsAndSuites));
-    }
-
-    // 5. Filter only.
-    if (!options.listOnly)
-      filterOnly(preprocessRoot);
-
-    // 6. Generate projects.
-    const fileSuites = new Map<string, Suite>();
-    for (const fileSuite of preprocessRoot.suites)
-      fileSuites.set(fileSuite._requireFile, fileSuite);
-
     const rootSuite = new Suite('', 'root');
-    for (const [project, files] of filesByProject) {
+    const runPhases = this._collectRunPhases(options);
-      const grepMatcher = createTitleMatcher(project.grep);
+    assert(runPhases.length > 0);
-      const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
+    for (const { projectFilter, testFileFilters } of runPhases) {
-      const projectSuite = new Suite(project.name, 'project');
+      // TODO: do not collect files for each project multiple times.
-      projectSuite._projectConfig = project;
+      const filesByProject = await this._collectFiles(testFileFilters, projectFilter);
-      if (project._fullyParallel)
+
-        projectSuite._parallelMode = 'parallel';
+      const allTestFiles = new Set<string>();
-      rootSuite._addSuite(projectSuite);
+      for (const files of filesByProject.values())
-      for (const file of files) {
+        files.forEach(file => allTestFiles.add(file));
-        const fileSuite = fileSuites.get(file);
+
-        if (!fileSuite)
+
-          continue;
+      // 1. Add all tests.
-        for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) {
+      const preprocessRoot = new Suite('', 'root');
-          const builtSuite = this._loader.buildFileSuiteForProject(project, fileSuite, repeatEachIndex, test => {
+      for (const file of allTestFiles) {
-            const grepTitle = test.titlePath().join(' ');
+        const fileSuite = await this._loader.loadTestFile(file, 'runner');
-            if (grepInvertMatcher?.(grepTitle))
+        if (fileSuite._loadError)
-              return false;
+          fatalErrors.push(fileSuite._loadError);
-            return grepMatcher(grepTitle);
+        preprocessRoot._addSuite(fileSuite);
-          });
+      }
-          if (builtSuite)
+
-            projectSuite._addSuite(builtSuite);
+      // 2. Complain about duplicate titles.
+      const duplicateTitlesError = createDuplicateTitlesError(config, preprocessRoot);
+      if (duplicateTitlesError)
+        fatalErrors.push(duplicateTitlesError);
+
+      // 3. Filter tests to respect line/column filter.
+      // TODO: figure out how this is supposed to work with groups.
+      filterByFocusedLine(preprocessRoot, testFileFilters);
+
+      // 4. Complain about only.
+      if (config.forbidOnly) {
+        const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
+        if (onlyTestsAndSuites.length > 0)
+          fatalErrors.push(createForbidOnlyError(config, onlyTestsAndSuites));
+      }
+
+      // 5. Filter only.
+      if (!options.listOnly)
+        filterOnly(preprocessRoot);
+
+      // 6. Generate projects.
+      const fileSuites = new Map<string, Suite>();
+      for (const fileSuite of preprocessRoot.suites)
+        fileSuites.set(fileSuite._requireFile, fileSuite);
+
+      const firstProjectSuiteIndex = rootSuite.suites.length;
+      for (const [project, files] of filesByProject) {
+        const grepMatcher = createTitleMatcher(project.grep);
+        const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
+        const projectSuite = new Suite(project.name, 'project');
+        projectSuite._projectConfig = project;
+        if (project._fullyParallel)
+          projectSuite._parallelMode = 'parallel';
+        rootSuite._addSuite(projectSuite);
+        for (const file of files) {
+          const fileSuite = fileSuites.get(file);
+          if (!fileSuite)
+            continue;
+          for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) {
+            const builtSuite = this._loader.buildFileSuiteForProject(project, fileSuite, repeatEachIndex, test => {
+              const grepTitle = test.titlePath().join(' ');
+              if (grepInvertMatcher?.(grepTitle))
+                return false;
+              return grepMatcher(grepTitle);
+            });
+            if (builtSuite)
+              projectSuite._addSuite(builtSuite);
+          }
         }
       }
+
+      const projectSuites = rootSuite.suites.slice(firstProjectSuiteIndex);
+      const testGroups = createTestGroups(projectSuites, config.workers);
+      concurrentTestGroups.push(testGroups);
     }
 
     // 7. Fail when no tests.
@@ -346,10 +418,10 @@ export class Runner {
       fatalErrors.push(createNoTestsError());
 
     // 8. Compute shards.
-    let testGroups = createTestGroups(rootSuite, config.workers);
-
     const shard = config.shard;
     if (shard) {
+      assert(!options.projectGroup);
+      assert(concurrentTestGroups.length === 1);
       const shardGroups: TestGroup[] = [];
       const shardTests = new Set<TestCase>();
 
@@ -362,7 +434,7 @@ export class Runner {
       const from = shardSize * currentShard + Math.min(extraOne, currentShard);
       const to = from + shardSize + (currentShard < extraOne ? 1 : 0);
       let current = 0;
-      for (const group of testGroups) {
+      for (const group of concurrentTestGroups[0]) {
         // Any test group goes to the shard that contains the first test of this group.
         // So, this shard gets any group that starts at [from; to)
         if (current >= from && current < to) {
@@ -373,11 +445,12 @@ export class Runner {
         current += group.tests.length;
       }
 
-      testGroups = shardGroups;
+      concurrentTestGroups[0] = shardGroups;
       filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test));
       total = rootSuite.allTests().length;
     }
-    config._testGroupsCount = testGroups.length;
+
+    config._maxConcurrentTestGroups = Math.max(...concurrentTestGroups.map(g => g.length));
 
     // 9. Report begin
     this._reporter.onBegin?.(config, rootSuite);
@@ -399,7 +472,7 @@ export class Runner {
 
     // 13. Run Global setup.
     const result: FullResult = { status: 'passed' };
-    const globalTearDown = await this._performGlobalSetup(config, rootSuite, [...filesByProject.keys()], result);
+    const globalTearDown = await this._performGlobalSetup(config, rootSuite, result);
     if (result.status !== 'passed')
       return result;
 
@@ -414,24 +487,32 @@ export class Runner {
 
     // 14. Run tests.
     try {
-      const sigintWatcher = new SigIntWatcher();
+      let sigintWatcher;
 
       let hasWorkerErrors = false;
-      const dispatcher = new Dispatcher(this._loader, testGroups, this._reporter);
+      for (const testGroups of concurrentTestGroups) {
-      await Promise.race([dispatcher.run(), sigintWatcher.promise()]);
+        const dispatcher = new Dispatcher(this._loader, testGroups, this._reporter);
-      if (!sigintWatcher.hadSignal()) {
+        sigintWatcher = new SigIntWatcher();
-        // We know for sure there was no Ctrl+C, so we remove custom SIGINT handler
+        await Promise.race([dispatcher.run(), sigintWatcher.promise()]);
-        // as soon as we can.
+        if (!sigintWatcher.hadSignal()) {
-        sigintWatcher.disarm();
+          // We know for sure there was no Ctrl+C, so we remove custom SIGINT handler
+          // as soon as we can.
+          sigintWatcher.disarm();
+        }
+        await dispatcher.stop();
+        hasWorkerErrors = dispatcher.hasWorkerErrors();
+        if (hasWorkerErrors)
+          break;
+        if (testGroups.some(testGroup => testGroup.tests.some(test => !test.ok())))
+          break;
+        if (sigintWatcher.hadSignal())
+          break;
       }
-      await dispatcher.stop();
+      if (sigintWatcher?.hadSignal()) {
-      hasWorkerErrors = dispatcher.hasWorkerErrors();
+        result.status = 'interrupted';
-
+      } else {
-      if (!sigintWatcher.hadSignal()) {
         const failed = hasWorkerErrors || rootSuite.allTests().some(test => !test.ok());
         result.status = failed ? 'failed' : 'passed';
-      } else {
-        result.status = 'interrupted';
       }
     } catch (e) {
       this._reporter.onError?.(serializeError(e));
@@ -457,7 +538,7 @@ export class Runner {
 
     // 4. Run Global setup.
     const result: FullResult = { status: 'passed' };
-    const globalTearDown = await this._performGlobalSetup(config, rootSuite, config.projects.filter(p => !options.projectFilter || options.projectFilter.includes(p.name)), result);
+    const globalTearDown = await this._performGlobalSetup(config, rootSuite, result);
     if (result.status !== 'passed')
       return result;
 
@@ -590,7 +671,7 @@ export class Runner {
     return true;
   }
 
-  private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, projects: FullProjectInternal[], result: FullResult): Promise<(() => Promise<void>) | undefined> {
+  private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, result: FullResult): Promise<(() => Promise<void>) | undefined> {
     let globalSetupResult: any = undefined;
 
     const pluginsThatWereSetUp: TestRunnerPlugin[] = [];
@@ -805,7 +886,7 @@ function buildItemLocation(rootDir: string, testOrSuite: Suite | TestCase) {
   return `${path.relative(rootDir, testOrSuite.location.file)}:${testOrSuite.location.line}`;
 }
 
-function createTestGroups(rootSuite: Suite, workers: number): TestGroup[] {
+function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[] {
   // This function groups tests that can be run together.
   // Tests cannot be run together when:
   // - They belong to different projects - requires different workers.
@@ -843,7 +924,7 @@ function createTestGroups(rootSuite: Suite, workers: number): TestGroup[] {
     };
   };
 
-  for (const projectSuite of rootSuite.suites) {
+  for (const projectSuite of projectSuites) {
     for (const test of projectSuite.allTests()) {
       let withWorkerHash = groups.get(test._workerHash);
       if (!withWorkerHash) {

packages/playwright-test/src/types.ts

@@ -44,7 +44,7 @@ export interface TestStepInternal {
 export interface FullConfigInternal extends FullConfigPublic {
   _globalOutputDir: string;
   _configDir: string;
-  _testGroupsCount: number;
+  _maxConcurrentTestGroups: number;
   _watchMode: boolean;
   _ignoreSnapshots: boolean;
   _workerIsolation: WorkerIsolation;
@@ -56,6 +56,14 @@ export interface FullConfigInternal extends FullConfigPublic {
 
   // Overrides the public field.
   projects: FullProjectInternal[];
+
+  groups?: { [key: string]: Array<string | Array<string | {
+    project: string | string[],
+    grep?: RegExp | RegExp[],
+    grepInvert?: RegExp | RegExp[],
+    testMatch?: string | RegExp | Array<string | RegExp>,
+    testIgnore?: string | RegExp | Array<string | RegExp>
+  }>> };
 }
 
 /**

packages/playwright-test/types/test.d.ts

@@ -687,6 +687,16 @@ interface TestConfig {
    */
   grepInvert?: RegExp|Array<RegExp>;
 
+  /**
+   * Project groups that control project execution order.
+   */
+  groups?: { [key: string]: Array<string|Array<string|{
+    /**
+     * Project name(s).
+     */
+    project: string|Array<string>;
+  }>>; };
+
   /**
    * Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await
    * expect(page).toHaveScreenshot()`.

tests/playwright-test/config.spec.ts

@@ -480,3 +480,51 @@ test('should have correct types for the config', async ({ runTSC }) => {
   });
   expect(result.exitCode).toBe(0);
 });
+
+test('should throw when group has duplicate project references', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          projects: [
+            { name: 'a' },
+          ],
+          groups: {
+            default: [
+              ['a', 'a']
+            ]
+          }
+        };
+    `,
+    'a.test.ts': `
+        const { test } = pwt;
+        test('pass', async () => {});
+      `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`config.groups.default group mentions project 'a' twice in one parallel group`);
+});
+
+test('should throw when group has unknown project reference', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          projects: [
+            { name: 'a' },
+          ],
+          groups: {
+            default: [
+              [{project: 'b'}]
+            ]
+          }
+        };
+    `,
+    'a.test.ts': `
+        const { test } = pwt;
+        test('pass', async () => {});
+      `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`config.groups.default refers to an unknown project 'b'`);
+});

tests/playwright-test/groups.spec.ts

@@ -0,0 +1,260 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { PlaywrightTestConfig, TestInfo } from '@playwright/test';
+import path from 'path';
+import { test, expect } from './playwright-test-fixtures';
+
+function createConfigWithProjects(names: string[], testInfo: TestInfo, groups: PlaywrightTestConfig['groups']): Record<string, string> {
+  const config: PlaywrightTestConfig = {
+    projects: names.map(name => ({ name, testDir: testInfo.outputPath(name) })),
+    groups
+  };
+  const files = {};
+  for (const name of names) {
+    files[`${name}/${name}.spec.ts`] = `
+      const { test } = pwt;
+      test('${name} test', async () => {
+        await new Promise(f => setTimeout(f, 100));
+      });`;
+  }
+  files['playwright.config.ts'] = `
+    import * as path from 'path';
+    module.exports = ${JSON.stringify(config)};
+    `;
+  return files;
+}
+
+type Timeline = { titlePath: string[], event: 'begin' | 'end' }[];
+
+function formatTimeline(timeline: Timeline) {
+  return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n');
+}
+
+function expectRunBefore(timeline: Timeline, before: string[], after: string[]) {
+  const begin = new Map<string, number>();
+  const end = new Map<string, number>();
+  for (let i = 0; i < timeline.length; i++) {
+    const projectName = timeline[i].titlePath[1];
+    const map = timeline[i].event === 'begin' ? begin : end;
+    const oldIndex = map.get(projectName) ?? i;
+    const newIndex = (timeline[i].event === 'begin') ? Math.min(i, oldIndex) : Math.max(i, oldIndex);
+    map.set(projectName, newIndex);
+  }
+  for (const b of before) {
+    for (const a of after) {
+      const bEnd = end.get(b) as number;
+      expect(bEnd === undefined, `Unknown project ${b}`).toBeFalsy();
+      const aBegin = begin.get(a) as number;
+      expect(aBegin === undefined, `Unknown project ${a}`).toBeFalsy();
+      if (bEnd < aBegin)
+        continue;
+      throw new Error(`Project '${b}' expected to finish before '${a}'\nTest run order was:\n${formatTimeline(timeline)}`);
+    }
+  }
+}
+
+test('should work', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: ['a']
+  });
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(1);
+  expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin]
+a > a${path.sep}a.spec.ts > a test [end]`);
+});
+
+test('should order two projects', async ({ runGroups }, testInfo) => {
+  await test.step(`order a then b`, async () => {
+    const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+      default: [
+        'a',
+        'b'
+      ]
+    });
+    const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(2);
+    expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin]
+a > a${path.sep}a.spec.ts > a test [end]
+b > b${path.sep}b.spec.ts > b test [begin]
+b > b${path.sep}b.spec.ts > b test [end]`);
+  });
+  await test.step(`order b then a`, async () => {
+    const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+      default: [
+        'b',
+        'a'
+      ]
+    });
+    const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(2);
+    expect(formatTimeline(timeline)).toEqual(`b > b${path.sep}b.spec.ts > b test [begin]
+b > b${path.sep}b.spec.ts > b test [end]
+a > a${path.sep}a.spec.ts > a test [begin]
+a > a${path.sep}a.spec.ts > a test [end]`);
+  });
+});
+
+test('should order 1-3-1 projects', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      'e',
+      ['d', 'c', 'b'],
+      'a',
+    ]
+  });
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expectRunBefore(timeline, ['e'], ['d', 'c', 'b']);
+  expectRunBefore(timeline, ['d', 'c', 'b'], ['a']);
+  expect(passed).toBe(5);
+});
+
+test('should order 2-2-2 projects', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      ['a', 'b'],
+      ['d', 'c'],
+      ['e', 'f'],
+    ]
+  });
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expectRunBefore(timeline, ['a', 'b'], ['c', 'd']);
+  expectRunBefore(timeline, ['c', 'd'], ['e', 'f']);
+  expect(passed).toBe(6);
+});
+
+test('should run parallel groups sequentially without overlaps', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      ['a', 'b', 'c', 'd'],
+      ['a', 'b', 'c', 'd'],
+      ['a', 'b', 'c', 'd'],
+    ]
+  });
+  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+
+  const expectedEndOfFirstPhase = events => {
+    const firstProjectEndIndex = project => events.findIndex(e => e.event === 'end' && e.titlePath[1] === project);
+    return Math.max(...['a', 'b', 'c', 'd'].map(firstProjectEndIndex));
+  };
+  const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event);
+
+  let remainingTimeline = timeline;
+  for (let i = 0; i < 3; i++) {
+    const phaseEndIndex = expectedEndOfFirstPhase(remainingTimeline);
+    const firstPhase = formatPhaseEvents(remainingTimeline.slice(0, phaseEndIndex + 1));
+    firstPhase.sort();
+    expect(firstPhase, `check phase ${i}`).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end', 'c:begin', 'c:end', 'd:begin', 'd:end']);
+    remainingTimeline = remainingTimeline.slice(phaseEndIndex + 1);
+  }
+  expect(remainingTimeline.length).toBe(0);
+
+  expect(passed).toBe(12);
+});
+
+test('should support phase with multiple project names', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      [
+        { project: ['a', 'b', 'c'] }
+      ],
+      [
+        { project: ['d'] },
+        { project: ['e', 'f'] }
+      ],
+    ]
+  });
+
+  const { exitCode, passed } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(6);
+});
+
+test('should support varios syntax', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      'a',
+      ['a', 'b'],
+      [
+        { project: ['a', 'b'] }
+      ],
+      [
+        { project: ['a', 'b'] },
+        'c',
+        { project: 'd' },
+      ],
+      [{ project: 'e' }],
+      'f'
+    ]
+  });
+  const { exitCode, passed } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(11);
+});
+
+test('should support --group option', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      'a', 'b'
+    ],
+    foo: [
+      ['b', 'c']
+    ],
+    bar: [
+      'd', 'e'
+    ]
+  });
+  const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event);
+  {
+    const { exitCode, passed, timeline } =  await runGroups(configWithFiles, { group: 'default' });
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(2);
+    expect(formatPhaseEvents(timeline)).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end']);
+  }
+  {
+    const { exitCode, passed, timeline } =  await runGroups(configWithFiles, { group: 'foo' });
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(2);
+    const formatted = formatPhaseEvents(timeline);
+    formatted.sort();
+    expect(formatted).toEqual(['b:begin', 'b:end', 'c:begin', 'c:end']);
+  }
+  {
+    const { exitCode, passed, timeline } =  await runGroups(configWithFiles, { group: 'bar' });
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(2);
+    expect(formatPhaseEvents(timeline)).toEqual(['d:begin', 'd:end', 'e:begin', 'e:end']);
+  }
+});
+
+test('should throw when unknown --group is passed', async ({ runGroups }, testInfo) => {
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
+    default: [
+      'a', 'b'
+    ],
+    foo: [
+      ['b', 'c']
+    ]
+  });
+  const { exitCode, output } =  await runGroups(configWithFiles, { group: 'bar' });
+  expect(exitCode).toBe(1);
+  expect(output).toContain(`Cannot find project group 'bar' in the config`);
+});

tests/playwright-test/playwright-test-fixtures.ts

@@ -224,7 +224,8 @@ type Fixtures = {
   writeFiles: (files: Files) => Promise<string>;
   runInlineTest: (files: Files, params?: Params, env?: Env, options?: RunOptions, beforeRunPlaywrightTest?: ({ baseDir }: { baseDir: string }) => Promise<void>) => Promise<RunResult>;
   runTSC: (files: Files) => Promise<TSCResult>;
-  nodeVersion: { major: number, minor: number, patch: number },
+  nodeVersion: { major: number, minor: number, patch: number };
+  runGroups: (files: Files, params?: Params, env?: Env, options?: RunOptions) => Promise<{ timeline: { titlePath: string[], event: 'begin' | 'end' }[] } & RunResult>;
 };
 
 export const test = base
@@ -261,6 +262,43 @@ export const test = base
         const [major, minor, patch] = process.versions.node.split('.');
         await use({ major: +major, minor: +minor, patch: +patch });
       },
+
+      runGroups: async ({ runInlineTest }, use, testInfo) => {
+        const timelinePath = testInfo.outputPath('timeline.json');
+        await use(async (files, params, env, options) => {
+          const result = await runInlineTest({
+            ...files,
+            'reporter.ts': `
+              import { Reporter, TestCase } from '@playwright/test/reporter';
+              import fs from 'fs';
+              import path from 'path';
+              class TimelineReporter implements Reporter {
+                private _timeline: {titlePath: string, event: 'begin' | 'end'}[] = [];
+                onTestBegin(test: TestCase) {
+                  this._timeline.push({ titlePath: test.titlePath(), event: 'begin' });
+                }
+                onTestEnd(test: TestCase) {
+                  this._timeline.push({ titlePath: test.titlePath(), event: 'end' });
+                }
+                onEnd() {
+                  fs.writeFileSync(path.join(${JSON.stringify(timelinePath)}), JSON.stringify(this._timeline, null, 2));
+                }
+              }
+              export default TimelineReporter;
+            `
+          }, { ...params, reporter: 'list,json,./reporter.ts', workers: 2 }, env, options);
+
+          let timeline;
+          try {
+            timeline = JSON.parse((await fs.promises.readFile(timelinePath, 'utf8')).toString('utf8'));
+          } catch (e) {
+          }
+          return {
+            ...result,
+            timeline
+          };
+        });
+      },
     });
 
 const TSCONFIG = {