diff --git a/docs/src/test-advanced.md b/docs/src/test-advanced.md index c178554fec..5757fe3521 100644 --- a/docs/src/test-advanced.md +++ b/docs/src/test-advanced.md @@ -37,8 +37,9 @@ These options would be typically different between local development and CI oper - `'never'` - do not preserve output for any tests; - `'failures-only'` - only preserve output for failed tests. - `projects: Project[]` - Multiple [projects](#projects) configuration. -- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit'` - The reporter to use. See [reporters](./test-reporters.md) for details. - `quiet: boolean` - Whether to suppress stdout and stderr from the tests. +- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit'` - The reporter to use. See [reporters](./test-reporters.md) for details. +- `reportSlowTests: { max: number, threshold: number } | null` - Whether to report slow tests. When `null`, slow tests are not reported. Otherwise, tests that took more than `threshold` milliseconds are reported as slow, but no more than `max` number of them. Passing zero as `max` reports all slow tests that exceed the threshold. - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shards) information. - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run. - `workers: number` - The maximum number of concurrent worker processes to use for parallelizing tests. diff --git a/src/test/cli.ts b/src/test/cli.ts index 34ae4298e1..fb89667630 100644 --- a/src/test/cli.ts +++ b/src/test/cli.ts @@ -30,6 +30,7 @@ const jsConfig = 'playwright.config.js'; const defaultConfig: Config = { preserveOutput: process.env.CI ? 'failures-only' : 'always', reporter: [ [defaultReporter] ], + reportSlowTests: { max: 5, threshold: 15000 }, timeout: defaultTimeout, updateSnapshots: process.env.CI ? 'none' : 'missing', workers: Math.ceil(require('os').cpus().length / 2), diff --git a/src/test/loader.ts b/src/test/loader.ts index 2ff92e4d30..2bce2b68b9 100644 --- a/src/test/loader.ts +++ b/src/test/loader.ts @@ -99,6 +99,7 @@ export class Loader { this._fullConfig.maxFailures = takeFirst(this._configOverrides.maxFailures, this._config.maxFailures, baseFullConfig.maxFailures); this._fullConfig.preserveOutput = takeFirst(this._configOverrides.preserveOutput, this._config.preserveOutput, baseFullConfig.preserveOutput); this._fullConfig.reporter = takeFirst(toReporters(this._configOverrides.reporter), toReporters(this._config.reporter), baseFullConfig.reporter); + this._fullConfig.reportSlowTests = takeFirst(this._configOverrides.reportSlowTests, this._config.reportSlowTests, baseFullConfig.reportSlowTests); this._fullConfig.quiet = takeFirst(this._configOverrides.quiet, this._config.quiet, baseFullConfig.quiet); this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard); this._fullConfig.updateSnapshots = takeFirst(this._configOverrides.updateSnapshots, this._config.updateSnapshots, baseFullConfig.updateSnapshots); @@ -295,6 +296,15 @@ function validateConfig(config: Config) { } } + if ('reportSlowTests' in config && config.reportSlowTests !== undefined && config.reportSlowTests !== null) { + if (!config.reportSlowTests || typeof config.reportSlowTests !== 'object') + throw new Error(`config.reportSlowTests must be an object`); + if (!('max' in config.reportSlowTests) || typeof config.reportSlowTests.max !== 'number' || config.reportSlowTests.max < 0) + throw new Error(`config.reportSlowTests.max must be a non-negative number`); + if (!('threshold' in config.reportSlowTests) || typeof config.reportSlowTests.threshold !== 'number' || config.reportSlowTests.threshold < 0) + throw new Error(`config.reportSlowTests.threshold must be a non-negative number`); + } + if ('shard' in config && config.shard !== undefined && config.shard !== null) { if (!config.shard || typeof config.shard !== 'object') throw new Error(`config.shard must be an object`); @@ -396,6 +406,7 @@ const baseFullConfig: FullConfig = { preserveOutput: 'always', projects: [], reporter: [ ['list'] ], + reportSlowTests: null, rootDir: path.resolve(process.cwd()), quiet: false, shard: null, diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts index 1b0be934ce..c189355fc4 100644 --- a/src/test/reporters/base.ts +++ b/src/test/reporters/base.ts @@ -21,7 +21,7 @@ import fs from 'fs'; import milliseconds from 'ms'; import path from 'path'; import StackUtils from 'stack-utils'; -import { FullConfig, TestStatus, Test, Suite, TestResult, TestError, Reporter } from '../reporter'; +import { FullConfig, TestStatus, Test, Spec, Suite, TestResult, TestError, Reporter } from '../reporter'; const stackUtils = new StackUtils(); @@ -56,10 +56,10 @@ export class BaseReporter implements Reporter { } onTestEnd(test: Test, result: TestResult) { - const spec = test.spec; - let duration = this.fileDurations.get(spec.file) || 0; - duration += result.duration; - this.fileDurations.set(spec.file, duration); + const relativePath = relativeSpecPath(this.config, test.spec); + const fileAndProject = relativePath + (test.projectName ? ` [${test.projectName}]` : ''); + const duration = this.fileDurations.get(fileAndProject) || 0; + this.fileDurations.set(fileAndProject, duration + result.duration); } onError(error: TestError) { @@ -75,14 +75,16 @@ export class BaseReporter implements Reporter { } private _printSlowTests() { + if (!this.config.reportSlowTests) + return; const fileDurations = [...this.fileDurations.entries()]; fileDurations.sort((a, b) => b[1] - a[1]); - for (let i = 0; i < 10 && i < fileDurations.length; ++i) { - const baseName = path.basename(fileDurations[i][0]); + const count = Math.min(fileDurations.length, this.config.reportSlowTests.max || Number.POSITIVE_INFINITY); + for (let i = 0; i < count; ++i) { const duration = fileDurations[i][1]; - if (duration < 15000) + if (duration <= this.config.reportSlowTests.threshold) break; - console.log(colors.yellow(' Slow test: ') + baseName + colors.yellow(` (${milliseconds(duration)})`)); + console.log(colors.yellow(' Slow test: ') + fileDurations[i][0] + colors.yellow(` (${milliseconds(duration)})`)); } } @@ -158,9 +160,13 @@ export function formatFailure(config: FullConfig, test: Test, index?: number): s return tokens.join('\n'); } +function relativeSpecPath(config: FullConfig, spec: Spec): string { + return path.relative(config.rootDir, spec.file) || path.basename(spec.file); +} + export function formatTestTitle(config: FullConfig, test: Test): string { const spec = test.spec; - let relativePath = path.relative(config.rootDir, spec.file) || path.basename(spec.file); + let relativePath = relativeSpecPath(config, spec); relativePath += ':' + spec.line + ':' + spec.column; return `${relativePath} › ${test.fullTitle()}`; } diff --git a/tests/playwright-test/base-reporter.spec.ts b/tests/playwright-test/base-reporter.spec.ts index 42837efe4f..2429f9a897 100644 --- a/tests/playwright-test/base-reporter.spec.ts +++ b/tests/playwright-test/base-reporter.spec.ts @@ -15,6 +15,7 @@ */ import { test, expect, stripAscii } from './playwright-test-fixtures'; +import * as path from 'path'; test('handle long test names', async ({ runInlineTest }) => { const title = 'title'.repeat(30); @@ -93,3 +94,67 @@ test('print an error in a codeframe', async ({ runInlineTest }) => { expect(result.output).toContain('throw error;'); expect(result.output).toContain('import myLib from \'./my-lib\';'); }); + +test('should print slow tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'foo' }, + { name: 'bar' }, + { name: 'baz' }, + { name: 'qux' }, + ], + reportSlowTests: { max: 0, threshold: 500 }, + }; + `, + 'dir/a.test.js': ` + const { test } = pwt; + test('slow test', async ({}) => { + await new Promise(f => setTimeout(f, 1000)); + }); + `, + 'dir/b.test.js': ` + const { test } = pwt; + test('fast test', async ({}) => { + await new Promise(f => setTimeout(f, 100)); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(8); + expect(stripAscii(result.output)).toContain(`Slow test: dir${path.sep}a.test.js [foo] (`); + expect(stripAscii(result.output)).toContain(`Slow test: dir${path.sep}a.test.js [bar] (`); + expect(stripAscii(result.output)).toContain(`Slow test: dir${path.sep}a.test.js [baz] (`); + expect(stripAscii(result.output)).toContain(`Slow test: dir${path.sep}a.test.js [qux] (`); + expect(stripAscii(result.output)).not.toContain(`Slow test: dir${path.sep}b.test.js [foo] (`); + expect(stripAscii(result.output)).not.toContain(`Slow test: dir${path.sep}b.test.js [bar] (`); + expect(stripAscii(result.output)).not.toContain(`Slow test: dir${path.sep}b.test.js [baz] (`); + expect(stripAscii(result.output)).not.toContain(`Slow test: dir${path.sep}b.test.js [qux] (`); +}); + +test('should not print slow tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'baz' }, + { name: 'qux' }, + ], + reportSlowTests: null, + }; + `, + 'dir/a.test.js': ` + const { test } = pwt; + test('slow test', async ({}) => { + await new Promise(f => setTimeout(f, 1000)); + }); + test('fast test', async ({}) => { + await new Promise(f => setTimeout(f, 100)); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(4); + expect(stripAscii(result.output)).not.toContain('Slow test'); +}); diff --git a/types/test.d.ts b/types/test.d.ts index de8c563fd7..4230f45105 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -29,6 +29,7 @@ export type ReporterDescription = [string] | [string, any]; export type Shard = { total: number, current: number } | null; +export type ReportSlowTests = { max: number, threshold: number } | null; export type PreserveOutput = 'always' | 'never' | 'failures-only'; export type UpdateSnapshots = 'all' | 'none' | 'missing'; @@ -162,6 +163,11 @@ interface ConfigBase { */ preserveOutput?: PreserveOutput; + /** + * Whether to suppress stdio output from the tests. + */ + quiet?: boolean; + /** * Reporter to use. Available options: * - `'list'` - default reporter, prints a single line per test; @@ -177,9 +183,12 @@ interface ConfigBase { reporter?: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'null' | ReporterDescription[]; /** - * Whether to suppress stdio output from the tests. + * Whether to report slow tests. When `null`, slow tests are not reported. + * Otherwise, tests that took more than `threshold` milliseconds are reported as slow, + * but no more than `max` number of them. Passing zero as `max` reports all slow tests + * that exceed the threshold. */ - quiet?: boolean; + reportSlowTests?: ReportSlowTests; /** * Shard tests and execute only the selected shard. @@ -218,6 +227,7 @@ export interface FullConfig { preserveOutput: PreserveOutput; projects: FullProject[]; reporter: ReporterDescription[]; + reportSlowTests: ReportSlowTests; rootDir: string; quiet: boolean; shard: Shard;