feat: config.globalScripts (#20062)

Introduce config.globalScripts. Tests from the matching files will run
before all projects. We'll only allow beforeAll/afterAll instead of
tests in such files (next PR).

Global scripts are executed as part of 'Global Scripts' project which is
not present in FullConfig.projects but may be referenced by
corresponding global setup Suites.

Signed-off-by: Yury Semikhatsky <yurys@chromium.org>
Co-authored-by: Dmitry Gozman <dgozman@gmail.com>

docs/src/test-api/class-testconfig.md

@@ -149,6 +149,18 @@ const config: PlaywrightTestConfig = {
 export default config;
 ```
 
+## property: TestConfig.globalScripts
+* since: v1.30
+- type: ?<[string]|[RegExp]|[Array]<[string]|[RegExp]>>
+
+Files that contain global setup/teardown hooks.
+
+**Details**
+
+[`method: Test.beforeAll`] hooks in the matching files will run before testing starts. [`method: Test.afterAll`] hooks in the matching files will run after testing finishes.
+
+If global setup fails, test execution will be skipped. [`method: Test.afterAll`] hooks will run in the same worker process as [`method: Test.beforeAll`].
+
 ## property: TestConfig.globalSetup
 * since: v1.10
 - type: ?<[string]>

packages/playwright-test/src/dispatcher.ts

@@ -32,7 +32,7 @@ export type TestGroup = {
   projectId: string;
   tests: TestCase[];
   watchMode: boolean;
-  isProjectSetup: boolean;
+  phase: 'test' | 'projectSetup' | 'globalSetup';
 };
 
 type TestResultData = {
@@ -573,7 +573,7 @@ class Worker extends EventEmitter {
         return { testId: test.id, retry: test.results.length };
       }),
       watchMode: testGroup.watchMode,
-      projectSetup: testGroup.isProjectSetup,
+      phase: testGroup.phase,
     };
     this.send({ method: 'run', params: runPayload });
   }

packages/playwright-test/src/ipc.ts

@@ -95,7 +95,7 @@ export type RunPayload = {
   file: string;
   entries: TestEntry[];
   watchMode: boolean;
-  projectSetup: boolean;
+  phase: 'test' | 'projectSetup' | 'globalSetup';
 };
 
 export type DonePayload = {

packages/playwright-test/src/loader.ts

@@ -138,6 +138,7 @@ export class Loader {
     this._fullConfig.shard = takeFirst(config.shard, baseFullConfig.shard);
     this._fullConfig._ignoreSnapshots = takeFirst(config.ignoreSnapshots, baseFullConfig._ignoreSnapshots);
     this._fullConfig.updateSnapshots = takeFirst(config.updateSnapshots, baseFullConfig.updateSnapshots);
+    this._fullConfig._globalScripts = takeFirst(config.globalScripts, null);
 
     const workers = takeFirst(config.workers, '50%');
     if (typeof workers === 'string') {
@@ -161,8 +162,9 @@ export class Loader {
       this._fullConfig._webServers = [webServers];
     }
     this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata);
+    this._fullConfig._globalProject = this._resolveProject(config, this._fullConfig, globalScriptsProject, throwawayArtifactsPath);
     this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath));
-    this._assignUniqueProjectIds(this._fullConfig.projects);
+    this._assignUniqueProjectIds([...this._fullConfig.projects, this._fullConfig._globalProject]);
   }
 
   private _assignUniqueProjectIds(projects: FullProjectInternal[]) {
@@ -180,12 +182,12 @@ export class Loader {
     }
   }
 
-  async loadTestFile(file: string, environment: 'runner' | 'worker', projectSetup: boolean) {
+  async loadTestFile(file: string, environment: 'runner' | 'worker', phase: 'test' | 'projectSetup' | 'globalSetup') {
     if (cachedFileSuites.has(file))
       return cachedFileSuites.get(file)!;
     const suite = new Suite(path.relative(this._fullConfig.rootDir, file) || path.basename(file), 'file');
     suite._requireFile = file;
-    suite._isProjectSetup = projectSetup;
+    suite._phase = phase;
     suite.location = { file, line: 0, column: 0 };
 
     setCurrentlyLoadingFileSuite(suite);
@@ -639,6 +641,11 @@ function validateProject(file: string, project: Project, title: string) {
   }
 }
 
+const globalScriptsProject: Project = {
+  name: 'Global Scripts',
+  repeatEach: 1,
+};
+
 export const baseFullConfig: FullConfigInternal = {
   forbidOnly: false,
   fullyParallel: false,
@@ -668,6 +675,8 @@ export const baseFullConfig: FullConfigInternal = {
   _maxConcurrentTestGroups: 0,
   _ignoreSnapshots: false,
   _workerIsolation: 'isolate-pools',
+  _globalScripts: null,
+  _globalProject: { } as FullProjectInternal,
 };
 
 function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined {

packages/playwright-test/src/runner.ts

@@ -238,13 +238,29 @@ export class Runner {
     return projects;
   }
 
-  private async _collectFiles(projects: FullProjectInternal[], commandLineFileFilters: TestFileFilter[]): Promise<{filesByProject: Map<FullProjectInternal, string[]>; setupFiles: Set<string>}> {
+  private async _collectFiles(projects: FullProjectInternal[], commandLineFileFilters: TestFileFilter[]): Promise<{filesByProject: Map<FullProjectInternal, string[]>; setupFiles: Set<string>, globalSetupFiles: Set<string>}> {
     const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
     const testFileExtension = (file: string) => extensions.includes(path.extname(file));
     const filesByProject = new Map<FullProjectInternal, string[]>();
     const setupFiles = new Set<string>();
     const fileToProjectName = new Map<string, string>();
     const commandLineFileMatcher = commandLineFileFilters.length ? createFileMatcherFromFilters(commandLineFileFilters) : () => true;
+
+    const config = this._loader.fullConfig();
+    const globalSetupFiles = new Set<string>();
+    if (config._globalScripts) {
+      const allFiles = await collectFiles(config.rootDir, true);
+      const globalScriptMatch = createFileMatcher(config._globalScripts);
+      const globalScripts = allFiles.filter(file => {
+        if (!testFileExtension(file) || !globalScriptMatch(file))
+          return false;
+        fileToProjectName.set(file, config._globalProject.name);
+        globalSetupFiles.add(file);
+        return true;
+      });
+      filesByProject.set(config._globalProject, globalScripts);
+    }
+
     for (const project of projects) {
       const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
       const setupMatch = createFileMatcher(project._setupMatch);
@@ -275,42 +291,45 @@ export class Runner {
       filesByProject.set(project, testFiles);
     }
 
-    return { filesByProject, setupFiles };
+    return { filesByProject, setupFiles, globalSetupFiles };
   }
 
-  private async _collectTestGroups(options: RunOptions): Promise<{ rootSuite: Suite, projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> {
+  private async _collectTestGroups(options: RunOptions): Promise<{ rootSuite: Suite, globalSetupGroups: TestGroup[], projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> {
     const config = this._loader.fullConfig();
     const projects = this._collectProjects(options.projectFilter);
-    const { filesByProject, setupFiles } = await this._collectFiles(projects, options.testFileFilters);
+    const { filesByProject, setupFiles, globalSetupFiles } = await this._collectFiles(projects, options.testFileFilters);
 
-    let result = await this._createFilteredRootSuite(options, filesByProject, new Set(), !!setupFiles.size, setupFiles);
+    let result = await this._createFilteredRootSuite(options, filesByProject, new Set(), !!setupFiles.size, setupFiles, globalSetupFiles);
     if (setupFiles.size) {
       const allTests = result.rootSuite.allTests();
-      const tests = allTests.filter(test => !test._isProjectSetup);
+      const tests = allTests.filter(test => test._phase === 'test');
       // If >0 tests match and
       // - none of the setup files match the filter then we run all setup files,
       // - if the filter also matches some of the setup tests, we'll run only
       //   that maching subset of setup tests.
       if (tests.length > 0 && tests.length === allTests.length)
-        result = await this._createFilteredRootSuite(options, filesByProject, setupFiles, false, setupFiles);
+        result = await this._createFilteredRootSuite(options, filesByProject, setupFiles, false, setupFiles, globalSetupFiles);
     }
 
     this._fatalErrors.push(...result.fatalErrors);
     const { rootSuite } = result;
 
     const allTestGroups = createTestGroups(rootSuite.suites, config.workers);
+    const globalSetupGroups = [];
     const projectSetupGroups = [];
     const testGroups = [];
     for (const group of allTestGroups) {
-      if (group.isProjectSetup)
+      if (group.phase === 'projectSetup')
         projectSetupGroups.push(group);
+      else if (group.phase === 'globalSetup')
+        globalSetupGroups.push(group);
       else
         testGroups.push(group);
     }
-    return { rootSuite, projectSetupGroups, testGroups };
+    return { rootSuite, globalSetupGroups, projectSetupGroups, testGroups };
   }
 
-  private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map<FullProjectInternal, string[]>, doNotFilterFiles: Set<string>, shouldCloneTests: boolean, setupFiles: Set<string>): Promise<{rootSuite: Suite, fatalErrors: TestError[]}> {
+  private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map<FullProjectInternal, string[]>, doNotFilterFiles: Set<string>, shouldCloneTests: boolean, setupFiles: Set<string>, globalSetupFiles: Set<string>): Promise<{rootSuite: Suite, fatalErrors: TestError[]}> {
     const config = this._loader.fullConfig();
     const fatalErrors: TestError[] = [];
     const allTestFiles = new Set<string>();
@@ -320,7 +339,12 @@ export class Runner {
     // Add all tests.
     const preprocessRoot = new Suite('', 'root');
     for (const file of allTestFiles) {
-      const fileSuite = await this._loader.loadTestFile(file, 'runner', setupFiles.has(file));
+      let type: 'test' | 'projectSetup' | 'globalSetup' = 'test';
+      if (globalSetupFiles.has(file))
+        type = 'globalSetup';
+      else if (setupFiles.has(file))
+        type = 'projectSetup';
+      const fileSuite = await this._loader.loadTestFile(file, 'runner', type);
       if (fileSuite._loadError)
         fatalErrors.push(fileSuite._loadError);
       // We have to clone only if there maybe subsequent calls of this method.
@@ -439,7 +463,10 @@ export class Runner {
       rootSuite.suites = [];
       rootSuite.tests = [];
     } else {
-      filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test));
+      // Unlike project setup files global setup always run regardless of the selected tests.
+      // Because of that we don't add global setup entries to shardTests to avoid running empty
+      // shards which have only global setup.
+      filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test) || test._phase === 'globalSetup');
     }
   }
 
@@ -447,7 +474,7 @@ export class Runner {
     const config = this._loader.fullConfig();
     // Each entry is an array of test groups that can be run concurrently. All
     // test groups from the previos entries must finish before entry starts.
-    const { rootSuite, projectSetupGroups, testGroups } = await this._collectTestGroups(options);
+    const { rootSuite, globalSetupGroups, projectSetupGroups, testGroups } = await this._collectTestGroups(options);
 
     // Fail when no tests.
     if (!rootSuite.allTests().length && !options.passWithNoTests)
@@ -455,7 +482,7 @@ export class Runner {
 
     this._filterForCurrentShard(rootSuite, projectSetupGroups, testGroups);
 
-    config._maxConcurrentTestGroups = Math.max(projectSetupGroups.length, testGroups.length);
+    config._maxConcurrentTestGroups = Math.max(globalSetupGroups.length, projectSetupGroups.length, testGroups.length);
 
     // Report begin
     this._reporter.onBegin?.(config, rootSuite);
@@ -492,15 +519,23 @@ export class Runner {
 
     // Run tests.
     try {
-      let dispatchResult = await this._dispatchToWorkers(projectSetupGroups);
+      // TODO: run only setups, keep workers alive, inherit process.env from global setup workers
+      let dispatchResult = await this._dispatchToWorkers(globalSetupGroups);
       if (dispatchResult === 'success') {
-        const failedSetupProjectIds = new Set<string>();
+        if (globalSetupGroups.some(group => group.tests.some(test => !test.ok()))) {
-        for (const testGroup of projectSetupGroups) {
+          this._skipTestsFromMatchingGroups([...testGroups, ...projectSetupGroups], () => true);
-          if (testGroup.tests.some(test => !test.ok()))
+        } else {
-            failedSetupProjectIds.add(testGroup.projectId);
+          dispatchResult = await this._dispatchToWorkers(projectSetupGroups);
+          if (dispatchResult === 'success') {
+            const failedSetupProjectIds = new Set<string>();
+            for (const testGroup of projectSetupGroups) {
+              if (testGroup.tests.some(test => !test.ok()))
+                failedSetupProjectIds.add(testGroup.projectId);
+            }
+            const testGroupsToRun = this._skipTestsFromMatchingGroups(testGroups, group => failedSetupProjectIds.has(group.projectId));
+            dispatchResult = await this._dispatchToWorkers(testGroupsToRun);
+          }
         }
-        const testGroupsToRun = this._skipTestsFromFailedProjects(testGroups, failedSetupProjectIds);
-        dispatchResult = await this._dispatchToWorkers(testGroupsToRun);
       }
       if (dispatchResult === 'signal') {
         result.status = 'interrupted';
@@ -534,10 +569,10 @@ export class Runner {
     return 'success';
   }
 
-  private _skipTestsFromFailedProjects(testGroups: TestGroup[], failedProjects: Set<string>): TestGroup[] {
+  private _skipTestsFromMatchingGroups(testGroups: TestGroup[], groupFilter: (g: TestGroup) => boolean): TestGroup[] {
     const result = [];
     for (const group of testGroups) {
-      if (failedProjects.has(group.projectId)) {
+      if (groupFilter(group)) {
         for (const test of group.tests) {
           const result = test._appendTestResult();
           this._reporter.onTestBegin?.(test, result);
@@ -823,7 +858,7 @@ function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[]
       projectId: test._projectId,
       tests: [],
       watchMode: false,
-      isProjectSetup: test._isProjectSetup,
+      phase: test._phase,
     };
   };
 

packages/playwright-test/src/test.ts

@@ -23,7 +23,7 @@ class Base {
   title: string;
   _only = false;
   _requireFile: string = '';
-  _isProjectSetup: boolean = false;
+  _phase: 'test' | 'projectSetup' | 'globalSetup' = 'test';
 
   constructor(title: string) {
     this.title = title;
@@ -121,7 +121,7 @@ export class Suite extends Base implements reporterTypes.Suite {
     suite._only = this._only;
     suite.location = this.location;
     suite._requireFile = this._requireFile;
-    suite._isProjectSetup = this._isProjectSetup;
+    suite._phase = this._phase;
     suite._use = this._use.slice();
     suite._hooks = this._hooks.slice();
     suite._timeout = this._timeout;
@@ -193,7 +193,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
     const test = new TestCase(this.title, this.fn, this._testType, this.location);
     test._only = this._only;
     test._requireFile = this._requireFile;
-    test._isProjectSetup = this._isProjectSetup;
+    test._phase = this._phase;
     test.expectedStatus = this.expectedStatus;
     test.annotations = this.annotations.slice();
     test._annotateWithInheritence = this._annotateWithInheritence;

packages/playwright-test/src/testType.ts

@@ -80,10 +80,10 @@ export class TestTypeImpl {
       ].join('\n'), location);
       return;
     }
-    if (allowedContext === 'projectSetup' && !suite._isProjectSetup)
+    if (allowedContext === 'projectSetup' && suite._phase !== 'projectSetup')
       addFatalError(`${title} is only allowed in a project setup file.`, location);
-    else if (allowedContext === 'test' && suite._isProjectSetup)
+    else if (allowedContext === 'test' && suite._phase !== 'test' && suite._phase !== 'globalSetup')
-      addFatalError(`${title} is not allowed in a project setup file.`, location);
+      addFatalError(`${title} is not allowed in a setup file.`, location);
     return suite;
   }
 
@@ -106,7 +106,7 @@ export class TestTypeImpl {
       return;
     const test = new TestCase(title, fn, this, location);
     test._requireFile = suite._requireFile;
-    test._isProjectSetup = suite._isProjectSetup;
+    test._phase = suite._phase;
     suite._addTest(test);
 
     if (type === 'only' || type === 'projectSetupOnly')
@@ -134,7 +134,7 @@ export class TestTypeImpl {
 
     const child = new Suite(title, 'describe');
     child._requireFile = suite._requireFile;
-    child._isProjectSetup = suite._isProjectSetup;
+    child._phase = suite._phase;
     child.location = location;
     suite._addSuite(child);
 

packages/playwright-test/src/types.ts

@@ -54,6 +54,10 @@ export interface FullConfigInternal extends FullConfigPublic {
    */
   webServer: FullConfigPublic['webServer'];
   _webServers: Exclude<FullConfigPublic['webServer'], null>[];
+  _globalScripts: string | RegExp | (string | RegExp)[] | null;
+
+  // This is an ephemeral project that is not added to `projects` list below.
+  _globalProject:  FullProjectInternal;
 
   // Overrides the public field.
   projects: FullProjectInternal[];

packages/playwright-test/src/workerRunner.ts

@@ -160,7 +160,11 @@ export class WorkerRunner extends EventEmitter {
       return;
 
     this._loader = await Loader.deserialize(this._params.loader);
-    this._project = this._loader.fullConfig().projects.find(p => p._id === this._params.projectId)!;
+    const globalProject = this._loader.fullConfig()._globalProject;
+    if (this._params.projectId === globalProject._id)
+      this._project = globalProject;
+    else
+      this._project = this._loader.fullConfig().projects.find(p => p._id === this._params.projectId)!;
   }
 
   async runTestGroup(runPayload: RunPayload) {
@@ -169,7 +173,7 @@ export class WorkerRunner extends EventEmitter {
     let fatalUnknownTestIds;
     try {
       await this._loadIfNeeded();
-      const fileSuite = await this._loader.loadTestFile(runPayload.file, 'worker', runPayload.projectSetup);
+      const fileSuite = await this._loader.loadTestFile(runPayload.file, 'worker', runPayload.phase);
       const suite = this._loader.buildFileSuiteForProject(this._project, fileSuite, this._params.repeatEachIndex, test => {
         if (runPayload.watchMode) {
           const testResolvedPayload: WatchTestResolvedPayload = {

packages/playwright-test/types/test.d.ts

@@ -646,6 +646,22 @@ interface TestConfig {
    */
   fullyParallel?: boolean;
 
+  /**
+   * Files that contain global setup/teardown hooks.
+   *
+   * **Details**
+   *
+   * [test.beforeAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-before-all) hooks in the matching
+   * files will run before testing starts.
+   * [test.afterAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-all) hooks in the matching
+   * files will run after testing finishes.
+   *
+   * If global setup fails, test execution will be skipped.
+   * [test.afterAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-all) hooks will run in the same
+   * worker process as [test.beforeAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-before-all).
+   */
+  globalScripts?: string|RegExp|Array<string|RegExp>;
+
   /**
    * Path to the global setup file. This file will be required and run before all the tests. It must export a single
    * function that takes a [`TestConfig`] argument.

tests/playwright-test/global-scripts.spec.ts

@@ -0,0 +1,225 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import path from 'path';
+import { test, expect } from './playwright-test-fixtures';
+
+type Timeline = { titlePath: string[], event: 'begin' | 'end' }[];
+
+function formatTimeline(timeline: Timeline) {
+  return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n');
+}
+
+function formatFileNames(timeline: Timeline) {
+  return timeline.map(e => e.titlePath[2]).join('\n');
+}
+
+function fileNames(timeline: Timeline) {
+  const fileNames = Array.from(new Set(timeline.map(({ titlePath }) => {
+    const name = titlePath[2];
+    const index = name.lastIndexOf(path.sep);
+    if (index === -1)
+      return name;
+    return name.slice(index + 1);
+  })).keys());
+  fileNames.sort();
+  return fileNames;
+}
+
+function expectFilesRunBefore(timeline: Timeline, before: string[], after: string[]) {
+  const fileBegin = name => {
+    const index = timeline.findIndex(({ titlePath }) => titlePath[2] === name);
+    expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1);
+    return index;
+  };
+  const fileEnd = name => {
+    // There is no Array.findLastIndex in Node < 18.
+    let index = -1;
+    for (index = timeline.length - 1; index >= 0; index--) {
+      if (timeline[index].titlePath[2] === name)
+        break;
+    }
+    expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1);
+    return index;
+  };
+
+  for (const b of before) {
+    const bEnd = fileEnd(b);
+    for (const a of after) {
+      const aBegin = fileBegin(a);
+      expect(bEnd < aBegin, `'${b}' expected to finish before ${a}, actual order:\n${formatTimeline(timeline)}`).toBeTruthy();
+    }
+  }
+}
+
+test('should work for one project', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        globalScripts: /.*global.ts/,
+        projects: [
+          {
+            name: 'p1',
+            testMatch: /.*.test.ts/,
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+    `,
+    'global.ts': `
+      const { test } = pwt;
+      test('setup1', async () => { });
+      test('setup2', async () => { });
+    `,
+  };
+  const { exitCode, passed, timeline } = await runGroups(files);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(4);
+  expect(formatTimeline(timeline)).toEqual(`Global Scripts > global.ts > setup1 [begin]
+Global Scripts > global.ts > setup1 [end]
+Global Scripts > global.ts > setup2 [begin]
+Global Scripts > global.ts > setup2 [end]
+p1 > a.test.ts > test1 [begin]
+p1 > a.test.ts > test1 [end]
+p1 > a.test.ts > test2 [begin]
+p1 > a.test.ts > test2 [end]`);
+});
+
+test('should work for several projects', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        globalScripts: /.*global.ts/,
+        projects: [
+          {
+            name: 'p1',
+            testMatch: /.*a.test.ts/,
+          },
+          {
+            name: 'p2',
+            testMatch: /.*b.test.ts/,
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+    `,
+    'b.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+    `,
+    'global.ts': `
+      const { test } = pwt;
+      test('setup1', async () => { });
+      test('setup2', async () => { });
+    `,
+  };
+  const { exitCode, passed, timeline } = await runGroups(files);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(6);
+  expectFilesRunBefore(timeline, [`global.ts`], [`a.test.ts`, `b.test.ts`]);
+});
+
+test('should skip tests if global setup fails', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        globalScripts: /.*global.ts/,
+        projects: [
+          {
+            name: 'p1',
+            testMatch: /.*a.test.ts/,
+          },
+          {
+            name: 'p2',
+            testMatch: /.*b.test.ts/,
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+    `,
+    'b.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+    `,
+    'global.ts': `
+      const { test, expect } = pwt;
+      test('setup1', async () => { });
+      test('setup2', async () => { expect(1).toBe(2) });
+    `,
+  };
+  const { exitCode, passed, skipped } = await runGroups(files);
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(1);
+  expect(skipped).toBe(3);
+});
+
+test('should run setup in each project shard', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        globalScripts: /.*global.ts/,
+        projects: [
+          {
+            name: 'p1',
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+      test('test3', async () => { });
+      test('test4', async () => { });
+    `,
+    'b.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+    `,
+    'global.ts': `
+      const { test, expect } = pwt;
+      test('setup1', async () => { });
+      test('setup2', async () => { });
+    `,
+  };
+
+  { // Shard 1/2
+    const { exitCode, passed, timeline, output } =  await runGroups(files, { shard: '1/2' });
+    expect(output).toContain('Running 6 tests using 1 worker, shard 1 of 2');
+    expect(fileNames(timeline)).toEqual(['a.test.ts', 'global.ts']);
+    expectFilesRunBefore(timeline, [`global.ts`], [`a.test.ts`]);
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(6);
+  }
+  { // Shard 2/2
+    const { exitCode, passed, timeline, output } =  await runGroups(files, { shard: '2/2' });
+    expect(output).toContain('Running 4 tests using 1 worker, shard 2 of 2');
+    expect(fileNames(timeline)).toEqual(['b.test.ts', 'global.ts']);
+    expectFilesRunBefore(timeline, [`global.ts`], [`b.test.ts`]);
+    expect(exitCode).toBe(0);
+    expect(passed).toBe(4);
+  }
+});
+

tests/playwright-test/project-setup.spec.ts

@@ -892,7 +892,7 @@ test('should prohibit beforeAll hooks in setup files', async ({ runGroups }, tes
 
   const { exitCode, output } =  await runGroups(files);
   expect(exitCode).toBe(1);
-  expect(output).toContain('test.beforeAll() is not allowed in a project setup file');
+  expect(output).toContain('test.beforeAll() is not allowed in a setup file');
 });
 
 test('should prohibit test in setup files', async ({ runGroups }, testInfo) => {
@@ -914,7 +914,7 @@ test('should prohibit test in setup files', async ({ runGroups }, testInfo) => {
 
   const { exitCode, output } =  await runGroups(files);
   expect(exitCode).toBe(1);
-  expect(output).toContain('test() is not allowed in a project setup file');
+  expect(output).toContain('test() is not allowed in a setup file');
 });
 
 test('should prohibit test hooks in setup files', async ({ runGroups }, testInfo) => {
@@ -936,5 +936,5 @@ test('should prohibit test hooks in setup files', async ({ runGroups }, testInfo
 
   const { exitCode, output } =  await runGroups(files);
   expect(exitCode).toBe(1);
-  expect(output).toContain('test.beforeEach() is not allowed in a project setup file');
+  expect(output).toContain('test.beforeEach() is not allowed in a setup file');
 });