Merge branch 'main' into sharding-algorithm

.github/workflows/tests_bidi.yml

@@ -42,3 +42,5 @@ jobs:
       if: matrix.channel == 'bidi-firefox-beta'
     - name: Run tests
       run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run biditest -- --project=${{ matrix.channel }}*
+      env:
+        PWTEST_USE_BIDI_EXPECTATIONS: '1'

.github/workflows/tests_secondary.yml

@@ -50,7 +50,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, macos-13, macos-14]
+        # Intel: macos-13, macos-14-large
+        # Arm64: macos-13-xlarge, macos-14
+        os: [macos-13, macos-13-xlarge, macos-14-large, macos-14]
         browser: [chromium, firefox, webkit]
     runs-on: ${{ matrix.os }}
     steps:
@@ -235,7 +237,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-12, windows-latest]
+        os: [ubuntu-20.04, macos-13, windows-latest]
     steps:
     - uses: actions/checkout@v4
     - uses: ./.github/actions/run-test

docs/src/browsers.md

@@ -459,9 +459,13 @@ Google Chrome and Microsoft Edge respect enterprise policies, which include limi
 
 Playwright's Firefox version matches the recent [Firefox Stable](https://www.mozilla.org/en-US/firefox/new/) build. Playwright doesn't work with the branded version of Firefox since it relies on patches.
 
+Note that availability of certain features, which depend heavily on the underlying platform, may vary between operating systems. For example, available media codecs vary substantially between Linux, macOS and Windows.
+
 ### WebKit
 
-Playwright's WebKit is derived from the latest WebKit main branch sources, often before these updates are incorporated into Apple Safari and other WebKit-based browsers. This gives a lot of lead time to react on the potential browser update issues. Playwright doesn't work with the branded version of Safari since it relies on patches. Instead, you can test using the most recent WebKit build. Note that availability of certain features, which depend heavily on the underlying platform, may vary between operating systems.
+Playwright's WebKit is derived from the latest WebKit main branch sources, often before these updates are incorporated into Apple Safari and other WebKit-based browsers. This gives a lot of lead time to react on the potential browser update issues. Playwright doesn't work with the branded version of Safari since it relies on patches. Instead, you can test using the most recent WebKit build.
+
+Note that availability of certain features, which depend heavily on the underlying platform, may vary between operating systems. For example, available media codecs vary substantially between Linux, macOS and Windows. While running WebKit on Linux CI is usually the most affordable option, for the closest-to-Safari experience you should run WebKit on mac, for example if you do video playback.
 
 ## Install behind a firewall or a proxy
 

packages/playwright-core/browsers.json

@@ -27,7 +27,7 @@
     },
     {
       "name": "webkit",
-      "revision": "2072",
+      "revision": "2073",
       "installByDefault": true,
       "revisionOverrides": {
         "mac10.14": "1446",

packages/playwright/src/runner/failureTracker.ts

@@ -31,7 +31,8 @@ export class FailureTracker {
   }
 
   onTestEnd(test: TestCase, result: TestResult) {
-    if (result.status !== 'skipped' && result.status !== test.expectedStatus)
+    // Test is considered failing after the last retry.
+    if (test.outcome() === 'unexpected' && test.results.length > test.retries)
       ++this._failureCount;
   }
 

packages/playwright/src/runner/runner.ts

@@ -15,12 +15,11 @@
  * limitations under the License.
  */
 
-import { monotonicTime } from 'playwright-core/lib/utils';
 import type { FullResult, TestError } from '../../types/testReporter';
 import { webServerPluginsForConfig } from '../plugins/webServerPlugin';
 import { collectFilesForProject, filterProjects } from './projectUtils';
 import { createErrorCollectingReporter, createReporters } from './reporters';
-import { TestRun, createTaskRunner, createTaskRunnerForClearCache, createTaskRunnerForDevServer, createTaskRunnerForList, createTaskRunnerForRelatedTestFiles } from './tasks';
+import { TestRun, createClearCacheTask, createGlobalSetupTasks, createLoadTask, createPluginSetupTasks, createReportBeginTask, createRunTestsTasks, createStartDevServerTask, runTasks } from './tasks';
 import type { FullConfigInternal } from '../common/config';
 import { affectedTestFiles } from '../transform/compilationCache';
 import { InternalReporter } from '../reporters/internalReporter';
@@ -69,7 +68,6 @@ export class Runner {
   async runAllTests(): Promise<FullResult['status']> {
     const config = this._config;
     const listOnly = config.cliListOnly;
-    const deadline = config.config.globalTimeout ? monotonicTime() + config.config.globalTimeout : 0;
 
     // Legacy webServer support.
     webServerPluginsForConfig(config).forEach(p => config.plugins.push({ factory: p }));
@@ -80,24 +78,15 @@ export class Runner {
       await lastRun.filterLastFailed();
 
     const reporter = new InternalReporter([...reporters, lastRun]);
-    const taskRunner = listOnly ? createTaskRunnerForList(
+    const tasks = listOnly ? [
-        config,
+      createLoadTask('in-process', { failOnLoadErrors: true, filterOnly: false }),
-        reporter,
+      createReportBeginTask(),
-        'in-process',
+    ] : [
-        { failOnLoadErrors: true }) : createTaskRunner(config, reporter);
+      ...createGlobalSetupTasks(config),
-
+      createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }),
-    const testRun = new TestRun(config);
+      ...createRunTestsTasks(config),
-    reporter.onConfigure(config.config);
+    ];
-
+    const status = await runTasks(new TestRun(config, reporter), tasks, config.config.globalTimeout);
-    const taskStatus = await taskRunner.run(testRun, deadline);
-    let status: FullResult['status'] = testRun.failureTracker.result();
-    if (status === 'passed' && taskStatus !== 'passed')
-      status = taskStatus;
-    const modifiedResult = await reporter.onEnd({ status });
-    if (modifiedResult && modifiedResult.status)
-      status = modifiedResult.status;
-
-    await reporter.onExit();
 
     // Calling process.exit() might truncate large stdout/stderr output.
     // See https://github.com/nodejs/node/issues/6456.
@@ -110,12 +99,10 @@ export class Runner {
   async findRelatedTestFiles(files: string[]): Promise<FindRelatedTestFilesReport>  {
     const errorReporter = createErrorCollectingReporter();
     const reporter = new InternalReporter([errorReporter]);
-    const taskRunner = createTaskRunnerForRelatedTestFiles(this._config, reporter, 'in-process', true);
+    const status = await runTasks(new TestRun(this._config, reporter), [
-    const testRun = new TestRun(this._config);
+      ...createPluginSetupTasks(this._config),
-    reporter.onConfigure(this._config.config);
+      createLoadTask('in-process', { failOnLoadErrors: true, filterOnly: false, populateDependencies: true }),
-    const status = await taskRunner.run(testRun, 0);
+    ]);
-    await reporter.onEnd({ status });
-    await reporter.onExit();
     if (status !== 'passed')
       return { errors: errorReporter.errors(), testFiles: [] };
     return { testFiles: affectedTestFiles(files) };
@@ -123,23 +110,21 @@ export class Runner {
 
   async runDevServer() {
     const reporter = new InternalReporter([createErrorCollectingReporter(true)]);
-    const taskRunner = createTaskRunnerForDevServer(this._config, reporter, 'in-process', true);
+    const status = await runTasks(new TestRun(this._config, reporter), [
-    const testRun = new TestRun(this._config);
+      ...createPluginSetupTasks(this._config),
-    reporter.onConfigure(this._config.config);
+      createLoadTask('in-process', { failOnLoadErrors: true, filterOnly: false }),
-    const status = await taskRunner.run(testRun, 0);
+      createStartDevServerTask(),
-    await reporter.onEnd({ status });
+      { title: 'wait until interrupted', setup: async () => new Promise(() => {}) },
-    await reporter.onExit();
+    ]);
     return { status };
   }
 
   async clearCache() {
     const reporter = new InternalReporter([createErrorCollectingReporter(true)]);
-    const taskRunner = createTaskRunnerForClearCache(this._config, reporter, 'in-process', true);
+    const status = await runTasks(new TestRun(this._config, reporter), [
-    const testRun = new TestRun(this._config);
+      ...createPluginSetupTasks(this._config),
-    reporter.onConfigure(this._config.config);
+      createClearCacheTask(this._config),
-    const status = await taskRunner.run(testRun, 0);
+    ]);
-    await reporter.onEnd({ status });
-    await reporter.onExit();
     return { status };
   }
 }

packages/playwright/src/runner/taskRunner.ts

@@ -19,31 +19,26 @@ import { ManualPromise, monotonicTime } from 'playwright-core/lib/utils';
 import type { FullResult, TestError } from '../../types/testReporter';
 import { SigIntWatcher } from './sigIntWatcher';
 import { serializeError } from '../util';
-import type { ReporterV2 } from '../reporters/reporterV2';
 import type { InternalReporter } from '../reporters/internalReporter';
 
-type TaskPhase<Context> = (reporter: ReporterV2, context: Context, errors: TestError[], softErrors: TestError[]) => Promise<void> | void;
+type TaskPhase<Context> = (context: Context, errors: TestError[], softErrors: TestError[]) => Promise<void> | void;
-export type Task<Context> = { setup?: TaskPhase<Context>, teardown?: TaskPhase<Context> };
+export type Task<Context> = { title: string, setup?: TaskPhase<Context>, teardown?: TaskPhase<Context> };
 
 export class TaskRunner<Context> {
-  private _tasks: { name: string, task: Task<Context> }[] = [];
+  private _tasks: Task<Context>[] = [];
   private _reporter: InternalReporter;
   private _hasErrors = false;
   private _interrupted = false;
   private _isTearDown = false;
   private _globalTimeoutForError: number;
 
-  static create<Context>(reporter: InternalReporter, globalTimeoutForError: number = 0) {
+  constructor(reporter: InternalReporter, globalTimeoutForError: number) {
-    return new TaskRunner<Context>(reporter, globalTimeoutForError);
-  }
-
-  private constructor(reporter: InternalReporter, globalTimeoutForError: number) {
     this._reporter = reporter;
     this._globalTimeoutForError = globalTimeoutForError;
   }
 
-  addTask(name: string, task: Task<Context>) {
+  addTask(task: Task<Context>) {
-    this._tasks.push({ name, task });
+    this._tasks.push(task);
   }
 
   async run(context: Context, deadline: number, cancelPromise?: ManualPromise<void>): Promise<FullResult['status']> {
@@ -61,18 +56,18 @@ export class TaskRunner<Context> {
     let currentTaskName: string | undefined;
 
     const taskLoop = async () => {
-      for (const { name, task } of this._tasks) {
+      for (const task of this._tasks) {
-        currentTaskName = name;
+        currentTaskName = task.title;
         if (this._interrupted)
           break;
-        debug('pw:test:task')(`"${name}" started`);
+        debug('pw:test:task')(`"${task.title}" started`);
         const errors: TestError[] = [];
         const softErrors: TestError[] = [];
         try {
-          teardownRunner._tasks.unshift({ name: `teardown for ${name}`, task: { setup: task.teardown } });
+          teardownRunner._tasks.unshift({ title: `teardown for ${task.title}`, setup: task.teardown });
-          await task.setup?.(this._reporter, context, errors, softErrors);
+          await task.setup?.(context, errors, softErrors);
         } catch (e) {
-          debug('pw:test:task')(`error in "${name}": `, e);
+          debug('pw:test:task')(`error in "${task.title}": `, e);
           errors.push(serializeError(e));
         } finally {
           for (const error of [...softErrors, ...errors])
@@ -83,7 +78,7 @@ export class TaskRunner<Context> {
             this._hasErrors = true;
           }
         }
-        debug('pw:test:task')(`"${name}" finished`);
+        debug('pw:test:task')(`"${task.title}" finished`);
       }
     };
 

packages/playwright/src/runner/tasks.ts

@@ -18,7 +18,7 @@ import fs from 'fs';
 import path from 'path';
 import { promisify } from 'util';
 import { debug } from 'playwright-core/lib/utilsBundle';
-import { removeFolders } from 'playwright-core/lib/utils';
+import { type ManualPromise, monotonicTime, removeFolders } from 'playwright-core/lib/utils';
 import { Dispatcher, type EnvByProjectId } from './dispatcher';
 import type { TestRunnerPluginRegistration } from '../plugins';
 import { createTestGroups, type TestGroup } from '../runner/testGroups';
@@ -33,6 +33,7 @@ import { FailureTracker } from './failureTracker';
 import { detectChangedTestFiles } from './vcs';
 import type { InternalReporter } from '../reporters/internalReporter';
 import { cacheDir } from '../transform/compilationCache';
+import type { FullResult } from '../../types/testReporter';
 
 const readDirAsync = promisify(fs.readdir);
 
@@ -42,132 +43,100 @@ type ProjectWithTestGroups = {
   testGroups: TestGroup[];
 };
 
-export type Phase = {
+type Phase = {
   dispatcher: Dispatcher,
   projects: ProjectWithTestGroups[]
 };
 
 export class TestRun {
   readonly config: FullConfigInternal;
+  readonly reporter: InternalReporter;
   readonly failureTracker: FailureTracker;
   rootSuite: Suite | undefined = undefined;
   readonly phases: Phase[] = [];
   projectFiles: Map<FullProjectInternal, string[]> = new Map();
   projectSuites: Map<FullProjectInternal, Suite[]> = new Map();
 
-  constructor(config: FullConfigInternal) {
+  constructor(config: FullConfigInternal, reporter: InternalReporter) {
     this.config = config;
+    this.reporter = reporter;
     this.failureTracker = new FailureTracker(config);
   }
 }
 
-export function createTaskRunner(config: FullConfigInternal, reporter: InternalReporter): TaskRunner<TestRun> {
+export async function runTasks(testRun: TestRun, tasks: Task<TestRun>[], globalTimeout?: number, cancelPromise?: ManualPromise<void>) {
-  const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
+  const deadline = globalTimeout ? monotonicTime() + globalTimeout : 0;
-  addGlobalSetupTasks(taskRunner, config);
+  const taskRunner = new TaskRunner<TestRun>(testRun.reporter, globalTimeout || 0);
-  taskRunner.addTask('load tests', createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }));
+  for (const task of tasks)
-  addRunTasks(taskRunner, config);
+    taskRunner.addTask(task);
-  return taskRunner;
+  testRun.reporter.onConfigure(testRun.config.config);
+  const status = await taskRunner.run(testRun, deadline, cancelPromise);
+  return await finishTaskRun(testRun, status);
 }
 
-export function createTaskRunnerForWatchSetup(config: FullConfigInternal, reporter: InternalReporter): TaskRunner<TestRun> {
+export async function runTasksDeferCleanup(testRun: TestRun, tasks: Task<TestRun>[]) {
-  const taskRunner = TaskRunner.create<TestRun>(reporter);
+  const taskRunner = new TaskRunner<TestRun>(testRun.reporter, 0);
-  addGlobalSetupTasks(taskRunner, config);
+  for (const task of tasks)
-  return taskRunner;
+    taskRunner.addTask(task);
+  testRun.reporter.onConfigure(testRun.config.config);
+  const { status, cleanup } = await taskRunner.runDeferCleanup(testRun, 0);
+  return { status: await finishTaskRun(testRun, status), cleanup };
 }
 
-export function createTaskRunnerForTestServer(config: FullConfigInternal, reporter: InternalReporter): TaskRunner<TestRun> {
+async function finishTaskRun(testRun: TestRun, status: FullResult['status']) {
-  const taskRunner = TaskRunner.create<TestRun>(reporter);
+  if (status === 'passed')
-  taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }));
+    status = testRun.failureTracker.result();
-  addRunTasks(taskRunner, config);
+  const modifiedResult = await testRun.reporter.onEnd({ status });
-  return taskRunner;
+  if (modifiedResult && modifiedResult.status)
+    status = modifiedResult.status;
+  await testRun.reporter.onExit();
+  return status;
 }
 
-function addGlobalSetupTasks(taskRunner: TaskRunner<TestRun>, config: FullConfigInternal) {
+export function createGlobalSetupTasks(config: FullConfigInternal) {
+  const tasks: Task<TestRun>[] = [];
   if (!config.configCLIOverrides.preserveOutputDir && !process.env.PW_TEST_NO_REMOVE_OUTPUT_DIRS)
-    taskRunner.addTask('clear output', createRemoveOutputDirsTask());
+    tasks.push(createRemoveOutputDirsTask());
-  for (const plugin of config.plugins)
+  tasks.push(...createPluginSetupTasks(config));
-    taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
   if (config.config.globalSetup || config.config.globalTeardown)
-    taskRunner.addTask('global setup', createGlobalSetupTask());
+    tasks.push(createGlobalSetupTask());
+  return tasks;
 }
 
-function addRunTasks(taskRunner: TaskRunner<TestRun>, config: FullConfigInternal) {
+export function createRunTestsTasks(config: FullConfigInternal) {
-  taskRunner.addTask('create phases', createPhasesTask());
+  return [
-  taskRunner.addTask('report begin', createReportBeginTask());
+    createPhasesTask(),
-  for (const plugin of config.plugins)
+    createReportBeginTask(),
-    taskRunner.addTask('plugin begin', createPluginBeginTask(plugin));
+    ...config.plugins.map(plugin => createPluginBeginTask(plugin)),
-  taskRunner.addTask('test suite', createRunTestsTask());
+    createRunTestsTask(),
-  return taskRunner;
+  ];
 }
 
-export function createTaskRunnerForList(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', options: { failOnLoadErrors: boolean }): TaskRunner<TestRun> {
+export function createClearCacheTask(config: FullConfigInternal): Task<TestRun> {
-  const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
+  return {
-  taskRunner.addTask('load tests', createLoadTask(mode, { ...options, filterOnly: false }));
+    title: 'clear cache',
-  taskRunner.addTask('report begin', createReportBeginTask());
-  return taskRunner;
-}
-
-export function createTaskRunnerForListFiles(config: FullConfigInternal, reporter: InternalReporter): TaskRunner<TestRun> {
-  const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
-  taskRunner.addTask('load tests', createListFilesTask());
-  taskRunner.addTask('report begin', createReportBeginTask());
-  return taskRunner;
-}
-
-export function createTaskRunnerForDevServer(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', setupAndWait: boolean): TaskRunner<TestRun> {
-  const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
-  if (setupAndWait) {
-    for (const plugin of config.plugins)
-      taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
-  }
-  taskRunner.addTask('load tests', createLoadTask(mode, { failOnLoadErrors: true, filterOnly: false }));
-  taskRunner.addTask('start dev server', createStartDevServerTask());
-  if (setupAndWait) {
-    taskRunner.addTask('wait until interrupted', {
-      setup: async () => new Promise(() => {}),
-    });
-  }
-  return taskRunner;
-}
-
-export function createTaskRunnerForRelatedTestFiles(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', setupPlugins: boolean): TaskRunner<TestRun> {
-  const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
-  if (setupPlugins) {
-    for (const plugin of config.plugins)
-      taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
-  }
-  taskRunner.addTask('load tests', createLoadTask(mode, { failOnLoadErrors: true, filterOnly: false, populateDependencies: true }));
-  return taskRunner;
-}
-
-export function createTaskRunnerForClearCache(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', setupPlugins: boolean): TaskRunner<TestRun> {
-  const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
-  if (setupPlugins) {
-    for (const plugin of config.plugins)
-      taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
-  }
-  taskRunner.addTask('clear cache', {
     setup: async () => {
       await removeDirAndLogToConsole(cacheDir);
       for (const plugin of config.plugins)
         await plugin.instance?.clearCache?.();
     },
-  });
+  };
-  return taskRunner;
 }
 
-function createReportBeginTask(): Task<TestRun> {
+export function createReportBeginTask(): Task<TestRun> {
   return {
-    setup: async (reporter, { rootSuite }) => {
+    title: 'report begin',
-      reporter.onBegin?.(rootSuite!);
+    setup: async testRun => {
+      testRun.reporter.onBegin?.(testRun.rootSuite!);
     },
     teardown: async ({}) => {},
   };
 }
 
-function createPluginSetupTask(plugin: TestRunnerPluginRegistration): Task<TestRun> {
+export function createPluginSetupTasks(config: FullConfigInternal): Task<TestRun>[] {
-  return {
+  return config.plugins.map(plugin => ({
-    setup: async (reporter, { config }) => {
+    title: 'plugin setup',
+    setup: async ({ reporter }) => {
       if (typeof plugin.factory === 'function')
         plugin.instance = await plugin.factory();
       else
@@ -177,13 +146,14 @@ function createPluginSetupTask(plugin: TestRunnerPluginRegistration): Task<TestR
     teardown: async () => {
       await plugin.instance?.teardown?.();
     },
-  };
+  }));
 }
 
 function createPluginBeginTask(plugin: TestRunnerPluginRegistration): Task<TestRun> {
   return {
-    setup: async (reporter, { rootSuite }) => {
+    title: 'plugin begin',
-      await plugin.instance?.begin?.(rootSuite!);
+    setup: async testRun => {
+      await plugin.instance?.begin?.(testRun.rootSuite!);
     },
     teardown: async () => {
       await plugin.instance?.end?.();
@@ -196,13 +166,14 @@ function createGlobalSetupTask(): Task<TestRun> {
   let globalSetupFinished = false;
   let teardownHook: any;
   return {
-    setup: async (reporter, { config }) => {
+    title: 'global setup',
+    setup: async ({ config }) => {
       const setupHook = config.config.globalSetup ? await loadGlobalHook(config, config.config.globalSetup) : undefined;
       teardownHook = config.config.globalTeardown ? await loadGlobalHook(config, config.config.globalTeardown) : undefined;
       globalSetupResult = setupHook ? await setupHook(config.config) : undefined;
       globalSetupFinished = true;
     },
-    teardown: async (reporter, { config }) => {
+    teardown: async ({ config }) => {
       if (typeof globalSetupResult === 'function')
         await globalSetupResult();
       if (globalSetupFinished)
@@ -213,7 +184,8 @@ function createGlobalSetupTask(): Task<TestRun> {
 
 function createRemoveOutputDirsTask(): Task<TestRun> {
   return {
-    setup: async (reporter, { config }) => {
+    title: 'clear output',
+    setup: async ({ config }) => {
       const outputDirs = new Set<string>();
       const projects = filterProjects(config.projects, config.cliProjectFilter);
       projects.forEach(p => outputDirs.add(p.project.outputDir));
@@ -235,9 +207,10 @@ function createRemoveOutputDirsTask(): Task<TestRun> {
   };
 }
 
-function createListFilesTask(): Task<TestRun> {
+export function createListFilesTask(): Task<TestRun> {
   return {
-    setup: async (reporter, testRun, errors) => {
+    title: 'load tests',
+    setup: async (testRun, errors) => {
       testRun.rootSuite = await createRootSuite(testRun, errors, false);
       testRun.failureTracker.onRootSuite(testRun.rootSuite);
       await collectProjectsAndTestFiles(testRun, false);
@@ -258,9 +231,10 @@ function createListFilesTask(): Task<TestRun> {
   };
 }
 
-function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, populateDependencies?: boolean }): Task<TestRun> {
+export function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, populateDependencies?: boolean }): Task<TestRun> {
   return {
-    setup: async (reporter, testRun, errors, softErrors) => {
+    title: 'load tests',
+    setup: async (testRun, errors, softErrors) => {
       await collectProjectsAndTestFiles(testRun, !!options.doNotRunDepsOutsideProjectFilter);
       await loadFileSuites(testRun, mode, options.failOnLoadErrors ? errors : softErrors);
 
@@ -294,7 +268,8 @@ function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filter
 
 function createPhasesTask(): Task<TestRun> {
   return {
-    setup: async (reporter, testRun) => {
+    title: 'create phases',
+    setup: async testRun => {
       let maxConcurrentTestGroups = 0;
 
       const processed = new Set<FullProjectInternal>();
@@ -325,7 +300,7 @@ function createPhasesTask(): Task<TestRun> {
           processed.add(project);
         if (phaseProjects.length) {
           let testGroupsInPhase = 0;
-          const phase: Phase = { dispatcher: new Dispatcher(testRun.config, reporter, testRun.failureTracker), projects: [] };
+          const phase: Phase = { dispatcher: new Dispatcher(testRun.config, testRun.reporter, testRun.failureTracker), projects: [] };
           testRun.phases.push(phase);
           for (const project of phaseProjects) {
             const projectSuite = projectToSuite.get(project)!;
@@ -345,7 +320,8 @@ function createPhasesTask(): Task<TestRun> {
 
 function createRunTestsTask(): Task<TestRun> {
   return {
-    setup: async (reporter, { phases, failureTracker }) => {
+    title: 'test suite',
+    setup: async ({ phases, failureTracker }) => {
       const successfulProjects = new Set<FullProjectInternal>();
       const extraEnvByProjectId: EnvByProjectId = new Map();
       const teardownToSetups = buildTeardownToSetupsMap(phases.map(phase => phase.projects.map(p => p.project)).flat());
@@ -389,28 +365,29 @@ function createRunTestsTask(): Task<TestRun> {
         }
       }
     },
-    teardown: async (reporter, { phases }) => {
+    teardown: async ({ phases }) => {
       for (const { dispatcher } of phases.reverse())
         await dispatcher.stop();
     },
   };
 }
 
-function createStartDevServerTask(): Task<TestRun> {
+export function createStartDevServerTask(): Task<TestRun> {
   return {
-    setup: async (reporter, testRun, errors, softErrors) => {
+    title: 'start dev server',
-      if (testRun.config.plugins.some(plugin => !!plugin.devServerCleanup)) {
+    setup: async ({ config }, errors, softErrors) => {
+      if (config.plugins.some(plugin => !!plugin.devServerCleanup)) {
         errors.push({ message: `DevServer is already running` });
         return;
       }
-      for (const plugin of testRun.config.plugins)
+      for (const plugin of config.plugins)
         plugin.devServerCleanup = await plugin.instance?.startDevServer?.();
-      if (!testRun.config.plugins.some(plugin => !!plugin.devServerCleanup))
+      if (!config.plugins.some(plugin => !!plugin.devServerCleanup))
         errors.push({ message: `DevServer is not available in the package you are using. Did you mean to use component testing?` });
     },
 
-    teardown: async (reporter, testRun) => {
+    teardown: async ({ config }) => {
-      for (const plugin of testRun.config.plugins) {
+      for (const plugin of config.plugins) {
         await plugin.devServerCleanup?.();
         plugin.devServerCleanup = undefined;
       }

packages/playwright/src/runner/testServer.ts

@@ -23,7 +23,7 @@ import type * as reporterTypes from '../../types/testReporter';
 import { affectedTestFiles, collectAffectedTestFiles, dependenciesForTestFile } from '../transform/compilationCache';
 import type { ConfigLocation, FullConfigInternal } from '../common/config';
 import { createErrorCollectingReporter, createReporterForTestServer, createReporters } from './reporters';
-import { TestRun, createTaskRunnerForList, createTaskRunnerForTestServer, createTaskRunnerForWatchSetup, createTaskRunnerForListFiles, createTaskRunnerForDevServer, createTaskRunnerForRelatedTestFiles, createTaskRunnerForClearCache } from './tasks';
+import { TestRun, runTasks, createLoadTask, createRunTestsTasks, createReportBeginTask, createListFilesTask, runTasksDeferCleanup, createClearCacheTask, createGlobalSetupTasks, createStartDevServerTask } from './tasks';
 import { open } from 'playwright-core/lib/utilsBundle';
 import ListReporter from '../reporters/list';
 import { SigIntWatcher } from './sigIntWatcher';
@@ -150,17 +150,13 @@ export class TestServerDispatcher implements TestServerInterface {
     if (!config)
       return { status: 'failed', report };
 
-    const taskRunner = createTaskRunnerForWatchSetup(config, reporter);
+    const { status, cleanup } = await runTasksDeferCleanup(new TestRun(config, reporter), [
-    reporter.onConfigure(config.config);
+      ...createGlobalSetupTasks(config),
-    const testRun = new TestRun(config);
+    ]);
-    const { status, cleanup: globalCleanup } = await taskRunner.runDeferCleanup(testRun, 0);
+    if (status !== 'passed')
-    await reporter.onEnd({ status });
+      await cleanup();
-    await reporter.onExit();
+    else
-    if (status !== 'passed') {
+      this._globalSetup = { cleanup, report };
-      await globalCleanup();
-      return { report, status };
-    }
-    this._globalSetup = { cleanup: globalCleanup, report };
     return { report, status };
   }
 
@@ -179,16 +175,13 @@ export class TestServerDispatcher implements TestServerInterface {
     if (!config)
       return { report, status: 'failed' };
 
-    const taskRunner = createTaskRunnerForDevServer(config, reporter, 'out-of-process', false);
+    const { status, cleanup } = await runTasksDeferCleanup(new TestRun(config, reporter), [
-    const testRun = new TestRun(config);
+      createLoadTask('out-of-process', { failOnLoadErrors: true, filterOnly: false }),
-    reporter.onConfigure(config.config);
+      createStartDevServerTask(),
-    const { status, cleanup } = await taskRunner.runDeferCleanup(testRun, 0);
+    ]);
-    await reporter.onEnd({ status });
+    if (status !== 'passed')
-    await reporter.onExit();
-    if (status !== 'passed') {
       await cleanup();
-      return { report, status };
+    else
-    }
       this._devServer = { cleanup, report };
     return { report, status };
   }
@@ -205,13 +198,9 @@ export class TestServerDispatcher implements TestServerInterface {
     const config = await this._loadConfigOrReportError(reporter);
     if (!config)
       return;
-
+    await runTasks(new TestRun(config, reporter), [
-    const taskRunner = createTaskRunnerForClearCache(config, reporter, 'out-of-process', false);
+      createClearCacheTask(config),
-    const testRun = new TestRun(config);
+    ]);
-    reporter.onConfigure(config.config);
-    const status = await taskRunner.run(testRun, 0);
-    await reporter.onEnd({ status });
-    await reporter.onExit();
   }
 
   async listFiles(params: Parameters<TestServerInterface['listFiles']>[0]): ReturnType<TestServerInterface['listFiles']> {
@@ -221,12 +210,10 @@ export class TestServerDispatcher implements TestServerInterface {
       return { status: 'failed', report };
 
     config.cliProjectFilter = params.projects?.length ? params.projects : undefined;
-    const taskRunner = createTaskRunnerForListFiles(config, reporter);
+    const status = await runTasks(new TestRun(config, reporter), [
-    reporter.onConfigure(config.config);
+      createListFilesTask(),
-    const testRun = new TestRun(config);
+      createReportBeginTask(),
-    const status = await taskRunner.run(testRun, 0);
+    ]);
-    await reporter.onEnd({ status });
-    await reporter.onExit();
     return { report, status };
   }
 
@@ -264,12 +251,10 @@ export class TestServerDispatcher implements TestServerInterface {
     config.cliProjectFilter = params.projects?.length ? params.projects : undefined;
     config.cliListOnly = true;
 
-    const taskRunner = createTaskRunnerForList(config, reporter, 'out-of-process', { failOnLoadErrors: false });
+    const status = await runTasks(new TestRun(config, reporter), [
-    const testRun = new TestRun(config);
+      createLoadTask('out-of-process', { failOnLoadErrors: false, filterOnly: false }),
-    reporter.onConfigure(config.config);
+      createReportBeginTask(),
-    const status = await taskRunner.run(testRun, 0);
+    ]);
-    await reporter.onEnd({ status });
-    await reporter.onExit();
     return { config, report, reporter, status };
   }
 
@@ -344,13 +329,12 @@ export class TestServerDispatcher implements TestServerInterface {
 
     const configReporters = await createReporters(config, 'test', true);
     const reporter = new InternalReporter([...configReporters, wireReporter]);
-    const taskRunner = createTaskRunnerForTestServer(config, reporter);
-    const testRun = new TestRun(config);
-    reporter.onConfigure(config.config);
     const stop = new ManualPromise();
-    const run = taskRunner.run(testRun, 0, stop).then(async status => {
+    const tasks = [
-      await reporter.onEnd({ status });
+      createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }),
-      await reporter.onExit();
+      ...createRunTestsTasks(config),
+    ];
+    const run = runTasks(new TestRun(config, reporter), tasks, 0, stop).then(async status => {
       this._testRun = undefined;
       return status;
     });
@@ -373,13 +357,9 @@ export class TestServerDispatcher implements TestServerInterface {
     const config = await this._loadConfigOrReportError(reporter);
     if (!config)
       return { errors: errorReporter.errors(), testFiles: [] };
-
+    const status = await runTasks(new TestRun(config, reporter), [
-    const taskRunner = createTaskRunnerForRelatedTestFiles(config, reporter, 'out-of-process', false);
+      createLoadTask('out-of-process', { failOnLoadErrors: true, filterOnly: false, populateDependencies: true }),
-    const testRun = new TestRun(config);
+    ]);
-    reporter.onConfigure(config.config);
-    const status = await taskRunner.run(testRun, 0);
-    await reporter.onEnd({ status });
-    await reporter.onExit();
     if (status !== 'passed')
       return { errors: errorReporter.errors(), testFiles: [] };
     return { testFiles: affectedTestFiles(params.files) };

tests/bidi/expectationReporter.ts

@@ -18,7 +18,8 @@ import type {
   FullConfig, FullResult, Reporter, Suite, TestCase
 } from '@playwright/test/reporter';
 import fs from 'fs';
-import { projectExpectationPath } from './expectationUtil';
+import { parseBidiExpectations as parseExpectations, projectExpectationPath } from './expectationUtil';
+import type { TestExpectation } from './expectationUtil';
 
 type ReporterOptions = {
   rebase?: boolean;
@@ -27,6 +28,7 @@ type ReporterOptions = {
 class ExpectationReporter implements Reporter {
   private _suite: Suite;
   private _options: ReporterOptions;
+  private _pendingUpdates: Promise<void>[] = [];
 
   constructor(options: ReporterOptions) {
     this._options = options;
@@ -40,18 +42,28 @@ class ExpectationReporter implements Reporter {
     if (!this._options.rebase)
       return;
     for (const project of this._suite.suites)
-      this._updateProjectExpectations(project);
+      this._pendingUpdates.push(this._updateProjectExpectations(project));
   }
 
-  private _updateProjectExpectations(project: Suite) {
+  async onExit() {
-    const results = project.allTests().map(test => {
+    await Promise.all(this._pendingUpdates);
-      const outcome = getOutcome(test);
+  }
-      const line = `${test.titlePath().slice(1).join(' › ')} [${outcome}]`;
+
-      return line;
+  private async _updateProjectExpectations(project: Suite) {
-    });
     const outputFile = projectExpectationPath(project.title);
+    const expectations = await parseExpectations(project.title);
+    for (const test of project.allTests()) {
+      const outcome = getOutcome(test);
+      // Strip root and project names.
+      const key = test.titlePath().slice(2).join(' › ');
+      if (!expectations.has(key) || expectations.get(key) === 'unknown')
+        expectations.set(key, outcome);
+    }
+    const keys = Array.from(expectations.keys());
+    keys.sort();
+    const results = keys.map(key => `${key} [${expectations.get(key)}]`);
     console.log('Writing new expectations to', outputFile);
-    fs.writeFileSync(outputFile, results.join('\n'));
+    await fs.promises.writeFile(outputFile, results.join('\n'));
   }
 
   printsToStdio(): boolean {
@@ -59,7 +71,7 @@ class ExpectationReporter implements Reporter {
   }
 }
 
-function getOutcome(test: TestCase): 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout' {
+function getOutcome(test: TestCase): TestExpectation {
   if (test.results.length === 0)
     return 'unknown';
   if (test.results.every(r => r.status === 'timedOut'))

tests/bidi/expectationUtil.ts

@@ -18,14 +18,27 @@ import fs from 'fs';
 import path from 'path';
 import type { TestInfo } from 'playwright/test';
 
+export type TestExpectation = 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout';
+
 type ShouldSkipPredicate = (info: TestInfo) => boolean;
 
-export async function parseBidiExpectations(projectName: string): Promise<ShouldSkipPredicate> {
+export async function createSkipTestPredicate(projectName: string): Promise<ShouldSkipPredicate> {
+  if (!process.env.PWTEST_USE_BIDI_EXPECTATIONS)
+    return () => false;
+  const expectationsMap = await parseBidiExpectations(projectName);
+  return (info: TestInfo) => {
+    const key = info.titlePath.join(' › ');
+    const expectation = expectationsMap.get(key);
+    return expectation === 'fail' || expectation === 'timeout';
+  };
+}
+
+export async function parseBidiExpectations(projectName: string): Promise<Map<string, TestExpectation>> {
   const filePath = projectExpectationPath(projectName);
   try {
     await fs.promises.access(filePath);
   } catch (e) {
-    return () => false;
+    return new Map();
   }
   const content = await fs.promises.readFile(filePath);
   const pairs = content.toString().split('\n').map(line => {
@@ -35,14 +48,8 @@ export async function parseBidiExpectations(projectName: string): Promise<Should
       return undefined;
     }
     return [match.groups!.titlePath, match.groups!.expectation];
-  }).filter(Boolean) as [string, string][];
+  }).filter(Boolean) as [string, TestExpectation][];
-  const expectationsMap = new Map(pairs);
+  return new Map(pairs);
-
-  return (info: TestInfo) => {
-    const key = [info.project.name, ...info.titlePath].join(' › ');
-    const expectation = expectationsMap.get(key);
-    return expectation === 'fail' || expectation === 'timeout';
-  };
 }
 
 export function projectExpectationPath(project: string): string {

tests/config/browserTest.ts

@@ -24,7 +24,7 @@ import { baseTest } from './baseTest';
 import { type RemoteServerOptions, type PlaywrightServer, RunServer, RemoteServer } from './remoteServer';
 import type { Log } from '../../packages/trace/src/har';
 import { parseHar } from '../config/utils';
-import { parseBidiExpectations as parseBidiProjectExpectations } from '../bidi/expectationUtil';
+import { createSkipTestPredicate } from '../bidi/expectationUtil';
 import type { TestInfo } from '@playwright/test';
 
 export type BrowserTestWorkerFixtures = PageWorkerFixtures & {
@@ -172,7 +172,7 @@ const test = baseTest.extend<BrowserTestTestFixtures, BrowserTestWorkerFixtures>
   },
 
   bidiTestSkipPredicate: [async ({ }, run) => {
-    const filter = await parseBidiProjectExpectations(test.info().project.name);
+    const filter = await createSkipTestPredicate(test.info().project.name);
     await run(filter);
   }, { scope: 'worker' }],
 

tests/playwright-test/max-failures.spec.ts

@@ -76,8 +76,8 @@ test('max-failures should work with retries', async ({ runInlineTest }) => {
     `,
   }, { 'max-failures': 2, 'retries': 4 });
   expect(result.exitCode).toBe(1);
-  expect(result.failed).toBe(1);
+  expect(result.failed).toBe(2);
-  expect(result.output.split('\n').filter(l => l.includes('Received:')).length).toBe(2);
+  expect(result.output.split('\n').filter(l => l.includes('Received:')).length).toBe(2 * (4 + 1)); // 2 tests * (4 retries + 1 original)
 });
 
 test('max-failures should stop workers', async ({ runInlineTest }) => {
@@ -181,3 +181,31 @@ test('max-failures should work across phases', async ({ runInlineTest }) => {
   expect(result.output).toContain('running c');
   expect(result.output).not.toContain('running d');
 });
+
+test('max-failures should not consider retries as failures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      export default {
+        maxFailures: 10,
+        retries: 10,
+      };
+    `,
+    'example.spec.ts': `
+      import { test, expect } from '@playwright/test';
+
+      test('I fail 9 times 1', () => {
+        if (test.info().retry < 9)
+          throw new Error('failing intentionally');
+      });
+
+      test('I fail 9 times 2', () => {
+        if (test.info().retry < 9)
+          throw new Error('failing intentionally');
+      });
+    `,
+  }, { workers: 1 });
+  expect(result.exitCode).toBe(0);
+  expect(result.failed).toBe(0);
+  expect(result.flaky).toBe(2);
+  expect(result.passed).toBe(0);
+});