From 7458c3292ceba0c11593428dccbfc45e91420dd5 Mon Sep 17 00:00:00 2001
From: Max Schmitt <max@schmitt.mx>
Date: Wed, 11 Sep 2024 20:35:26 +0200
Subject: [PATCH] fix(test-runner): do not consider retries for maxFailures
 (#32533)

Fixes https://github.com/microsoft/playwright/issues/26350
---
 .../playwright/src/runner/failureTracker.ts   |  3 +-
 tests/playwright-test/max-failures.spec.ts    | 32 +++++++++++++++++--
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.ts
index 6ea8f81a34..ac62677571 100644
--- a/packages/playwright/src/runner/failureTracker.ts
+++ b/packages/playwright/src/runner/failureTracker.ts
@@ -31,7 +31,8 @@ export class FailureTracker {
   }
 
   onTestEnd(test: TestCase, result: TestResult) {
-    if (result.status !== 'skipped' && result.status !== test.expectedStatus)
+    // Test is considered failing after the last retry.
+    if (test.outcome() === 'unexpected' && test.results.length > test.retries)
       ++this._failureCount;
   }
 
diff --git a/tests/playwright-test/max-failures.spec.ts b/tests/playwright-test/max-failures.spec.ts
index c3bf4af834..fed320b826 100644
--- a/tests/playwright-test/max-failures.spec.ts
+++ b/tests/playwright-test/max-failures.spec.ts
@@ -76,8 +76,8 @@ test('max-failures should work with retries', async ({ runInlineTest }) => {
     `,
   }, { 'max-failures': 2, 'retries': 4 });
   expect(result.exitCode).toBe(1);
-  expect(result.failed).toBe(1);
-  expect(result.output.split('\n').filter(l => l.includes('Received:')).length).toBe(2);
+  expect(result.failed).toBe(2);
+  expect(result.output.split('\n').filter(l => l.includes('Received:')).length).toBe(2 * (4 + 1)); // 2 tests * (4 retries + 1 original)
 });
 
 test('max-failures should stop workers', async ({ runInlineTest }) => {
@@ -181,3 +181,31 @@ test('max-failures should work across phases', async ({ runInlineTest }) => {
   expect(result.output).toContain('running c');
   expect(result.output).not.toContain('running d');
 });
+
+test('max-failures should not consider retries as failures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      export default {
+        maxFailures: 10,
+        retries: 10,
+      };
+    `,
+    'example.spec.ts': `
+      import { test, expect } from '@playwright/test';
+
+      test('I fail 9 times 1', () => {
+        if (test.info().retry < 9)
+          throw new Error('failing intentionally');
+      });
+
+      test('I fail 9 times 2', () => {
+        if (test.info().retry < 9)
+          throw new Error('failing intentionally');
+      });
+    `,
+  }, { workers: 1 });
+  expect(result.exitCode).toBe(0);
+  expect(result.failed).toBe(0);
+  expect(result.flaky).toBe(2);
+  expect(result.passed).toBe(0);
+});