diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.ts index 6ea8f81a34..03c4c7121b 100644 --- a/packages/playwright/src/runner/failureTracker.ts +++ b/packages/playwright/src/runner/failureTracker.ts @@ -31,8 +31,12 @@ export class FailureTracker { } onTestEnd(test: TestCase, result: TestResult) { - if (result.status !== 'skipped' && result.status !== test.expectedStatus) + if (result.status === 'skipped') + return; + if (result.status !== test.expectedStatus) ++this._failureCount; + else + this._failureCount -= result.retry; } onWorkerError() { diff --git a/tests/playwright-test/max-failures.spec.ts b/tests/playwright-test/max-failures.spec.ts index c3bf4af834..157b18ee2d 100644 --- a/tests/playwright-test/max-failures.spec.ts +++ b/tests/playwright-test/max-failures.spec.ts @@ -181,3 +181,40 @@ test('max-failures should work across phases', async ({ runInlineTest }) => { expect(result.output).toContain('running c'); expect(result.output).not.toContain('running d'); }); + +test('max-failures should not consider retries as failures', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { + maxFailures: 10, + retries: 10, + }; + `, + 'example.spec.ts': ` + import fs from 'fs'; + import { test, expect } from '@playwright/test'; + + test('I fail 9 times 1', () => { + let count = parseInt(fs.readFileSync('count1', 'utf8'), 10); + count++; + fs.writeFileSync('count1', String(count)); + if (count < 10) + throw new Error('failing intentionally'); + }); + + test('I fail 9 times 2', () => { + let count = parseInt(fs.readFileSync('count2', 'utf8'), 10); + count++; + fs.writeFileSync('count2', String(count)); + if (count < 10) + throw new Error('failing intentionally'); + }); + `, + 'count1': '0', + 'count2': '0', + }, { workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.failed).toBe(0); + expect(result.flaky).toBe(2); + expect(result.passed).toBe(0); +});