From 1d786c804dbdb6ce1882a98c91b3ee889de96482 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Mon, 22 Apr 2024 14:00:16 -0700 Subject: [PATCH] chore: generate globalTimeout and maxFailures errors in runner (#30467) Generate 'global timeout' and 'max failures' errors in the runner. Avoid reading `config.globalTimeout` and `config.maxFailures` in the base reporters. Reference https://github.com/microsoft/playwright/issues/29768 --- packages/playwright/src/reporters/base.ts | 11 -------- packages/playwright/src/runner/dispatcher.ts | 7 ++++- .../playwright/src/runner/failureTracker.ts | 7 +++-- packages/playwright/src/runner/taskRunner.ts | 4 +-- tests/playwright-test/reporter-base.spec.ts | 26 +++++++++++++++++++ 5 files changed, 39 insertions(+), 16 deletions(-) diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index fff2703208..0f1f7bb0ce 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -194,8 +194,6 @@ export class BaseReporter implements ReporterV2 { tokens.push(colors.yellow(` ${didNotRun} did not run`)); if (expected) tokens.push(colors.green(` ${expected} passed`) + colors.dim(` (${milliseconds(this.result.duration)})`)); - if (this.result.status === 'timedout') - tokens.push(colors.red(` Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`)); if (fatalErrors.length && expected + unexpected.length + interrupted.length + flaky.length > 0) tokens.push(colors.red(` ${fatalErrors.length === 1 ? '1 error was not a part of any test' : fatalErrors.length + ' errors were not a part of any test'}, see above for details`)); @@ -250,7 +248,6 @@ export class BaseReporter implements ReporterV2 { if (full && summary.failuresToPrint.length && !this._omitFailures) this._printFailures(summary.failuresToPrint); this._printSlowTests(); - this._printMaxFailuresReached(); this._printSummary(summaryMessage); } @@ -272,14 +269,6 @@ export class BaseReporter implements ReporterV2 { console.log(colors.yellow(' Consider splitting slow test files to speed up parallel execution')); } - private _printMaxFailuresReached() { - if (!this.config.maxFailures) - return; - if (this._failureCount < this.config.maxFailures) - return; - console.log(colors.yellow(`Testing stopped early after ${this.config.maxFailures} maximum allowed failures.`)); - } - private _printSummary(summary: string) { if (summary.trim()) console.log(summary); diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 850e3d4232..ad44ce453e 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -26,6 +26,7 @@ import type { TestGroup } from './testGroups'; import type { FullConfigInternal } from '../common/config'; import type { ReporterV2 } from '../reporters/reporterV2'; import type { FailureTracker } from './failureTracker'; +import { colors } from 'playwright-core/lib/utilsBundle'; export type EnvByProjectId = Map>; @@ -540,9 +541,13 @@ class JobDispatcher { private _reportTestEnd(test: TestCase, result: TestResult) { this._reporter.onTestEnd(test, result); + const hadMaxFailures = this._failureTracker.hasReachedMaxFailures(); this._failureTracker.onTestEnd(test, result); - if (this._failureTracker.hasReachedMaxFailures()) + if (this._failureTracker.hasReachedMaxFailures()) { this._stopCallback(); + if (!hadMaxFailures) + this._reporter.onError({ message: colors.red(`Testing stopped early after ${this._failureTracker.maxFailures()} maximum allowed failures.`) }); + } } } diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.ts index eb39b1d44f..702ea668c2 100644 --- a/packages/playwright/src/runner/failureTracker.ts +++ b/packages/playwright/src/runner/failureTracker.ts @@ -40,8 +40,7 @@ export class FailureTracker { } hasReachedMaxFailures() { - const maxFailures = this._config.config.maxFailures; - return maxFailures > 0 && this._failureCount >= maxFailures; + return this.maxFailures() > 0 && this._failureCount >= this.maxFailures(); } hasWorkerErrors() { @@ -51,4 +50,8 @@ export class FailureTracker { result(): 'failed' | 'passed' { return this._hasWorkerErrors || this.hasReachedMaxFailures() || this._rootSuite?.allTests().some(test => !test.ok()) ? 'failed' : 'passed'; } + + maxFailures() { + return this._config.config.maxFailures; + } } diff --git a/packages/playwright/src/runner/taskRunner.ts b/packages/playwright/src/runner/taskRunner.ts index 5312cb2f7e..d49b1c4295 100644 --- a/packages/playwright/src/runner/taskRunner.ts +++ b/packages/playwright/src/runner/taskRunner.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { debug } from 'playwright-core/lib/utilsBundle'; +import { colors, debug } from 'playwright-core/lib/utilsBundle'; import { ManualPromise, monotonicTime } from 'playwright-core/lib/utils'; import type { FullResult, TestError } from '../../types/testReporter'; import { SigIntWatcher } from './sigIntWatcher'; @@ -99,7 +99,7 @@ export class TaskRunner { if (sigintWatcher.hadSignal() || cancelPromise?.isDone()) { status = 'interrupted'; } else if (timeoutWatcher.timedOut()) { - this._reporter.onError?.({ message: `Timed out waiting ${this._globalTimeoutForError / 1000}s for the ${currentTaskName} to run` }); + this._reporter.onError?.({ message: colors.red(`Timed out waiting ${this._globalTimeoutForError / 1000}s for the ${currentTaskName} to run`) }); status = 'timedout'; } else if (this._hasErrors) { status = 'failed'; diff --git a/tests/playwright-test/reporter-base.spec.ts b/tests/playwright-test/reporter-base.spec.ts index 8c8a2bc67d..ca4801cc83 100644 --- a/tests/playwright-test/reporter-base.spec.ts +++ b/tests/playwright-test/reporter-base.spec.ts @@ -209,6 +209,32 @@ for (const useIntermediateMergeReport of [false, true] as const) { expect(result.output).toContain('Testing stopped early after 1 maximum allowed failures.'); }); + test('should print if globalTimeout is reached', async ({ runInlineTest }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/29768' }); + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + globalTimeout: 1000, + }; + `, + 'dir/a.test.js': ` + import { test, expect } from '@playwright/test'; + test('first', async ({}) => { + }); + test('second (hanging)', async ({}) => { + await new Promise(() => {}); + }); + test('third', async ({}) => { + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(1); + expect(result.interrupted).toBe(1); + expect(result.didNotRun).toBe(1); + expect(result.output).toContain('Timed out waiting 1s for the test suite to run'); + }); + test('should not print slow parallel tests', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.ts': `