diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.ts index 6ea8f81a34..ac62677571 100644 --- a/packages/playwright/src/runner/failureTracker.ts +++ b/packages/playwright/src/runner/failureTracker.ts @@ -31,7 +31,8 @@ export class FailureTracker { } onTestEnd(test: TestCase, result: TestResult) { - if (result.status !== 'skipped' && result.status !== test.expectedStatus) + // Test is considered failing after the last retry. + if (test.outcome() === 'unexpected' && test.results.length > test.retries) ++this._failureCount; } diff --git a/tests/playwright-test/max-failures.spec.ts b/tests/playwright-test/max-failures.spec.ts index c3bf4af834..fed320b826 100644 --- a/tests/playwright-test/max-failures.spec.ts +++ b/tests/playwright-test/max-failures.spec.ts @@ -76,8 +76,8 @@ test('max-failures should work with retries', async ({ runInlineTest }) => { `, }, { 'max-failures': 2, 'retries': 4 }); expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.output.split('\n').filter(l => l.includes('Received:')).length).toBe(2); + expect(result.failed).toBe(2); + expect(result.output.split('\n').filter(l => l.includes('Received:')).length).toBe(2 * (4 + 1)); // 2 tests * (4 retries + 1 original) }); test('max-failures should stop workers', async ({ runInlineTest }) => { @@ -181,3 +181,31 @@ test('max-failures should work across phases', async ({ runInlineTest }) => { expect(result.output).toContain('running c'); expect(result.output).not.toContain('running d'); }); + +test('max-failures should not consider retries as failures', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { + maxFailures: 10, + retries: 10, + }; + `, + 'example.spec.ts': ` + import { test, expect } from '@playwright/test'; + + test('I fail 9 times 1', () => { + if (test.info().retry < 9) + throw new Error('failing intentionally'); + }); + + test('I fail 9 times 2', () => { + if (test.info().retry < 9) + throw new Error('failing intentionally'); + }); + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.failed).toBe(0); + expect(result.flaky).toBe(2); + expect(result.passed).toBe(0); +});