feat(test runner): configurable reportSlowTests (#7120)

Also splits tests by projects and reports them with nice relative paths.

docs/src/test-advanced.md

@@ -37,8 +37,9 @@ These options would be typically different between local development and CI oper
   - `'never'` - do not preserve output for any tests;
   - `'failures-only'` - only preserve output for failed tests.
 - `projects: Project[]` - Multiple [projects](#projects) configuration.
-- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit'` - The reporter to use. See [reporters](./test-reporters.md) for details.
 - `quiet: boolean` - Whether to suppress stdout and stderr from the tests.
+- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit'` - The reporter to use. See [reporters](./test-reporters.md) for details.
+- `reportSlowTests: { max: number, threshold: number } | null` - Whether to report slow tests. When `null`, slow tests are not reported. Otherwise, tests that took more than `threshold` milliseconds are reported as slow, but no more than `max` number of them. Passing zero as `max` reports all slow tests that exceed the threshold.
 - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shards) information.
 - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run.
 - `workers: number` - The maximum number of concurrent worker processes to use for parallelizing tests.

src/test/cli.ts

@@ -30,6 +30,7 @@ const jsConfig = 'playwright.config.js';
 const defaultConfig: Config = {
   preserveOutput: process.env.CI ? 'failures-only' : 'always',
   reporter: [ [defaultReporter] ],
+  reportSlowTests: { max: 5, threshold: 15000 },
   timeout: defaultTimeout,
   updateSnapshots: process.env.CI ? 'none' : 'missing',
   workers: Math.ceil(require('os').cpus().length / 2),

src/test/loader.ts

@@ -99,6 +99,7 @@ export class Loader {
     this._fullConfig.maxFailures = takeFirst(this._configOverrides.maxFailures, this._config.maxFailures, baseFullConfig.maxFailures);
     this._fullConfig.preserveOutput = takeFirst<PreserveOutput>(this._configOverrides.preserveOutput, this._config.preserveOutput, baseFullConfig.preserveOutput);
     this._fullConfig.reporter = takeFirst(toReporters(this._configOverrides.reporter), toReporters(this._config.reporter), baseFullConfig.reporter);
+    this._fullConfig.reportSlowTests = takeFirst(this._configOverrides.reportSlowTests, this._config.reportSlowTests, baseFullConfig.reportSlowTests);
     this._fullConfig.quiet = takeFirst(this._configOverrides.quiet, this._config.quiet, baseFullConfig.quiet);
     this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard);
     this._fullConfig.updateSnapshots = takeFirst(this._configOverrides.updateSnapshots, this._config.updateSnapshots, baseFullConfig.updateSnapshots);
@@ -295,6 +296,15 @@ function validateConfig(config: Config) {
     }
   }
 
+  if ('reportSlowTests' in config && config.reportSlowTests !== undefined && config.reportSlowTests !== null) {
+    if (!config.reportSlowTests || typeof config.reportSlowTests !== 'object')
+      throw new Error(`config.reportSlowTests must be an object`);
+    if (!('max' in config.reportSlowTests) || typeof config.reportSlowTests.max !== 'number' || config.reportSlowTests.max < 0)
+      throw new Error(`config.reportSlowTests.max must be a non-negative number`);
+    if (!('threshold' in config.reportSlowTests) || typeof config.reportSlowTests.threshold !== 'number' || config.reportSlowTests.threshold < 0)
+      throw new Error(`config.reportSlowTests.threshold must be a non-negative number`);
+  }
+
   if ('shard' in config && config.shard !== undefined && config.shard !== null) {
     if (!config.shard || typeof config.shard !== 'object')
       throw new Error(`config.shard must be an object`);
@@ -396,6 +406,7 @@ const baseFullConfig: FullConfig = {
   preserveOutput: 'always',
   projects: [],
   reporter: [ ['list'] ],
+  reportSlowTests: null,
   rootDir: path.resolve(process.cwd()),
   quiet: false,
   shard: null,

src/test/reporters/base.ts

@@ -21,7 +21,7 @@ import fs from 'fs';
 import milliseconds from 'ms';
 import path from 'path';
 import StackUtils from 'stack-utils';
-import { FullConfig, TestStatus, Test, Suite, TestResult, TestError, Reporter } from '../reporter';
+import { FullConfig, TestStatus, Test, Spec, Suite, TestResult, TestError, Reporter } from '../reporter';
 
 const stackUtils = new StackUtils();
 
@@ -56,10 +56,10 @@ export class BaseReporter implements Reporter  {
   }
 
   onTestEnd(test: Test, result: TestResult) {
-    const spec = test.spec;
-    let duration = this.fileDurations.get(spec.file) || 0;
-    duration += result.duration;
-    this.fileDurations.set(spec.file, duration);
+    const relativePath = relativeSpecPath(this.config, test.spec);
+    const fileAndProject = relativePath + (test.projectName ? ` [${test.projectName}]` : '');
+    const duration = this.fileDurations.get(fileAndProject) || 0;
+    this.fileDurations.set(fileAndProject, duration + result.duration);
   }
 
   onError(error: TestError) {
@@ -75,14 +75,16 @@ export class BaseReporter implements Reporter  {
   }
 
   private _printSlowTests() {
+    if (!this.config.reportSlowTests)
+      return;
     const fileDurations = [...this.fileDurations.entries()];
     fileDurations.sort((a, b) => b[1] - a[1]);
-    for (let i = 0; i < 10 && i < fileDurations.length; ++i) {
-      const baseName = path.basename(fileDurations[i][0]);
+    const count = Math.min(fileDurations.length, this.config.reportSlowTests.max || Number.POSITIVE_INFINITY);
+    for (let i = 0; i < count; ++i) {
       const duration = fileDurations[i][1];
-      if (duration < 15000)
+      if (duration <= this.config.reportSlowTests.threshold)
         break;
-      console.log(colors.yellow('  Slow test: ') + baseName + colors.yellow(` (${milliseconds(duration)})`));
+      console.log(colors.yellow('  Slow test: ') + fileDurations[i][0] + colors.yellow(` (${milliseconds(duration)})`));
     }
   }
 
@@ -158,9 +160,13 @@ export function formatFailure(config: FullConfig, test: Test, index?: number): s
   return tokens.join('\n');
 }
 
+function relativeSpecPath(config: FullConfig, spec: Spec): string {
+  return path.relative(config.rootDir, spec.file) || path.basename(spec.file);
+}
+
 export function formatTestTitle(config: FullConfig, test: Test): string {
   const spec = test.spec;
-  let relativePath = path.relative(config.rootDir, spec.file) || path.basename(spec.file);
+  let relativePath = relativeSpecPath(config, spec);
   relativePath += ':' + spec.line + ':' + spec.column;
   return `${relativePath} › ${test.fullTitle()}`;
 }

tests/playwright-test/base-reporter.spec.ts

@@ -15,6 +15,7 @@
  */
 
 import { test, expect, stripAscii } from './playwright-test-fixtures';
+import * as path from 'path';
 
 test('handle long test names', async ({ runInlineTest }) => {
   const title = 'title'.repeat(30);
@@ -93,3 +94,67 @@ test('print an error in a codeframe', async ({ runInlineTest }) => {
   expect(result.output).toContain('throw error;');
   expect(result.output).toContain('import myLib from \'./my-lib\';');
 });
+
+test('should print slow tests', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'foo' },
+          { name: 'bar' },
+          { name: 'baz' },
+          { name: 'qux' },
+        ],
+        reportSlowTests: { max: 0, threshold: 500 },
+      };
+    `,
+    'dir/a.test.js': `
+      const { test } = pwt;
+      test('slow test', async ({}) => {
+        await new Promise(f => setTimeout(f, 1000));
+      });
+    `,
+    'dir/b.test.js': `
+      const { test } = pwt;
+      test('fast test', async ({}) => {
+        await new Promise(f => setTimeout(f, 100));
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(8);
+  expect(stripAscii(result.output)).toContain(`Slow test: dir${path.sep}a.test.js [foo] (`);
+  expect(stripAscii(result.output)).toContain(`Slow test: dir${path.sep}a.test.js [bar] (`);
+  expect(stripAscii(result.output)).toContain(`Slow test: dir${path.sep}a.test.js [baz] (`);
+  expect(stripAscii(result.output)).toContain(`Slow test: dir${path.sep}a.test.js [qux] (`);
+  expect(stripAscii(result.output)).not.toContain(`Slow test: dir${path.sep}b.test.js [foo] (`);
+  expect(stripAscii(result.output)).not.toContain(`Slow test: dir${path.sep}b.test.js [bar] (`);
+  expect(stripAscii(result.output)).not.toContain(`Slow test: dir${path.sep}b.test.js [baz] (`);
+  expect(stripAscii(result.output)).not.toContain(`Slow test: dir${path.sep}b.test.js [qux] (`);
+});
+
+test('should not print slow tests', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'baz' },
+          { name: 'qux' },
+        ],
+        reportSlowTests: null,
+      };
+    `,
+    'dir/a.test.js': `
+      const { test } = pwt;
+      test('slow test', async ({}) => {
+        await new Promise(f => setTimeout(f, 1000));
+      });
+      test('fast test', async ({}) => {
+        await new Promise(f => setTimeout(f, 100));
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(4);
+  expect(stripAscii(result.output)).not.toContain('Slow test');
+});

types/test.d.ts

@@ -29,6 +29,7 @@ export type ReporterDescription =
   [string] | [string, any];
 
 export type Shard = { total: number, current: number } | null;
+export type ReportSlowTests = { max: number, threshold: number } | null;
 export type PreserveOutput = 'always' | 'never' | 'failures-only';
 export type UpdateSnapshots = 'all' | 'none' | 'missing';
 
@@ -162,6 +163,11 @@ interface ConfigBase {
    */
   preserveOutput?: PreserveOutput;
 
+  /**
+   * Whether to suppress stdio output from the tests.
+   */
+   quiet?: boolean;
+
   /**
    * Reporter to use. Available options:
    * - `'list'` - default reporter, prints a single line per test;
@@ -177,9 +183,12 @@ interface ConfigBase {
   reporter?: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'null' | ReporterDescription[];
 
   /**
-   * Whether to suppress stdio output from the tests.
+   * Whether to report slow tests. When `null`, slow tests are not reported.
+   * Otherwise, tests that took more than `threshold` milliseconds are reported as slow,
+   * but no more than `max` number of them. Passing zero as `max` reports all slow tests
+   * that exceed the threshold.
    */
-  quiet?: boolean;
+  reportSlowTests?: ReportSlowTests;
 
   /**
    * Shard tests and execute only the selected shard.
@@ -218,6 +227,7 @@ export interface FullConfig {
   preserveOutput: PreserveOutput;
   projects: FullProject[];
   reporter: ReporterDescription[];
+  reportSlowTests: ReportSlowTests;
   rootDir: string;
   quiet: boolean;
   shard: Shard;