feat(test): more information about timing out fixtures (#13546)

- Always show a fixture that was running during timeout.
- Give custom titles to built-in fixtures.
- Specify setup/teardown fixture phase in the message.
- Split connect vs launch browser fixtures for better naming.

Example timeout message:

```log
Timeout of 2000ms exceeded while running fixture "built-in playwright configuration" teardown.
```

packages/playwright-test/src/fixtures.ts

@@ -31,6 +31,7 @@ type FixtureRegistration = {
   fn: Function | any;  // Either a fixture function, or a fixture value.
   auto: boolean;
   option: boolean;
+  customTitle?: string;
   timeout?: number;
   deps: string[];  // Names of the dependencies, ({ foo, bar }) => {...}
   id: string;  // Unique id, to differentiate between fixtures with the same name.
@@ -54,7 +55,7 @@ class Fixture {
     this.usages = new Set();
     this.value = null;
     this._runnableDescription = {
-      fixture: this.registration.name,
+      title: `fixture "${this.registration.customTitle || this.registration.name}" setup`,
       location: registration.location,
       slot: this.registration.timeout === undefined ? undefined : {
         timeout: this.registration.timeout,
@@ -117,6 +118,7 @@ class Fixture {
       this.usages.clear();
       if (this._useFuncFinished) {
         debugTest(`teardown ${this.registration.name}`);
+        this._runnableDescription.title = `fixture "${this.registration.customTitle || this.registration.name}" teardown`;
         timeoutManager.setCurrentFixture(this._runnableDescription);
         this._useFuncFinished.resolve();
         await this._selfTeardownComplete;
@@ -147,13 +149,14 @@ export class FixturePool {
       for (const entry of Object.entries(fixtures)) {
         const name = entry[0];
         let value = entry[1];
-        let options: { auto: boolean, scope: FixtureScope, option: boolean, timeout: number | undefined } | undefined;
+        let options: { auto: boolean, scope: FixtureScope, option: boolean, timeout: number | undefined, customTitle: string | undefined } | undefined;
         if (isFixtureTuple(value)) {
           options = {
             auto: !!value[1].auto,
             scope: value[1].scope || 'test',
             option: !!value[1].option,
             timeout: value[1].timeout,
+            customTitle: (value[1] as any)._title,
           };
           value = value[0];
         }
@@ -166,9 +169,9 @@ export class FixturePool {
           if (previous.auto !== options.auto)
             throw errorWithLocations(`Fixture "${name}" has already been registered as a { auto: '${previous.scope}' } fixture.`, { location, name }, previous);
         } else if (previous) {
-          options = { auto: previous.auto, scope: previous.scope, option: previous.option, timeout: previous.timeout };
+          options = { auto: previous.auto, scope: previous.scope, option: previous.option, timeout: previous.timeout, customTitle: previous.customTitle };
         } else if (!options) {
-          options = { auto: false, scope: 'test', option: false, timeout: undefined };
+          options = { auto: false, scope: 'test', option: false, timeout: undefined, customTitle: undefined };
         }
 
         if (options.scope !== 'test' && options.scope !== 'worker')
@@ -177,7 +180,7 @@ export class FixturePool {
           throw errorWithLocations(`Cannot use({ ${name} }) in a describe group, because it forces a new worker.\nMake it top-level in the test file or put in the configuration file.`, { location, name });
 
         const deps = fixtureParameterNames(fn, location);
-        const registration: FixtureRegistration = { id: '', name, location, scope: options.scope, fn, auto: options.auto, option: options.option, timeout: options.timeout, deps, super: previous };
+        const registration: FixtureRegistration = { id: '', name, location, scope: options.scope, fn, auto: options.auto, option: options.option, timeout: options.timeout, customTitle: options.customTitle, deps, super: previous };
         registrationId(registration);
         this.registrations.set(name, registration);
       }

packages/playwright-test/src/index.ts

@@ -16,7 +16,7 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext, Video, APIRequestContext, Tracing } from 'playwright-core';
+import type { LaunchOptions, BrowserContextOptions, Page, Browser, BrowserContext, Video, APIRequestContext, Tracing } from 'playwright-core';
 import type { TestType, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, TestInfo } from '../types/test';
 import { rootTestType } from './testType';
 import { createGuid, debugMode } from 'playwright-core/lib/utils';
@@ -44,6 +44,7 @@ type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & {
   _contextFactory: (options?: BrowserContextOptions) => Promise<BrowserContext>;
 };
 type WorkerFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & {
+  _connectedBrowser: Browser | undefined,
   _browserOptions: LaunchOptions;
   _artifactsDir: () => string;
   _snapshotSuffix: string;
@@ -86,12 +87,12 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
     });
     if (dir)
       await removeFolders([dir]);
-  }, { scope: 'worker' }],
+  }, { scope: 'worker', _title: 'built-in playwright configuration' } as any],
 
   _browserOptions: [async ({ playwright, headless, channel, launchOptions }, use) => {
     const options: LaunchOptions = {
       handleSIGINT: false,
-      timeout: 30000,  // 30 seconds
+      timeout: 0,
       ...launchOptions,
     };
     if (headless !== undefined)
@@ -106,27 +107,37 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
       (browserType as any)._defaultLaunchOptions = undefined;
   }, { scope: 'worker', auto: true }],
 
-  browser: [async ({ playwright, browserName, channel, headless, connectOptions }, use) => {
+  _connectedBrowser: [async ({ playwright, browserName, channel, headless, connectOptions }, use) => {
+    if (!connectOptions) {
+      await use(undefined);
+      return;
+    }
     if (!['chromium', 'firefox', 'webkit'].includes(browserName))
       throw new Error(`Unexpected browserName "${browserName}", must be one of "chromium", "firefox" or "webkit"`);
-    if (connectOptions) {
+    const browser = await playwright[browserName].connect(connectOptions.wsEndpoint, {
-      const browser = await playwright[browserName].connect(connectOptions.wsEndpoint, {
+      headers: {
-        headers: {
+        'x-playwright-browser': channel || browserName,
-          'x-playwright-browser': channel || browserName,
+        'x-playwright-headless': headless ? '1' : '0',
-          'x-playwright-headless': headless ? '1' : '0',
+        ...connectOptions.headers,
-          ...connectOptions.headers,
+      },
-        },
+      timeout: connectOptions.timeout ?? 3 * 60 * 1000, // 3 minutes
-        timeout: connectOptions.timeout ?? 3 * 60 * 1000, // 3 minutes
+    });
-      });
+    await use(browser);
-      await use(browser);
+    await browser.close();
-      await browser.close();
+  }, { scope: 'worker', timeout: 0, _title: 'remote connection' } as any],
+
+  browser: [async ({ playwright, browserName, _connectedBrowser }, use) => {
+    if (_connectedBrowser) {
+      await use(_connectedBrowser);
       return;
     }
 
+    if (!['chromium', 'firefox', 'webkit'].includes(browserName))
+      throw new Error(`Unexpected browserName "${browserName}", must be one of "chromium", "firefox" or "webkit"`);
     const browser = await playwright[browserName].launch();
     await use(browser);
     await browser.close();
-  }, { scope: 'worker', timeout: 0 } ],
+  }, { scope: 'worker' } ],
 
   acceptDownloads: [ undefined, { option: true } ],
   bypassCSP: [ undefined, { option: true } ],
@@ -422,9 +433,9 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
       else
         await fs.promises.unlink(file).catch(() => {});
     }));
-  }, { auto: true }],
+  }, { auto: true,  _title: 'built-in playwright configuration' } as any],
 
-  _contextFactory: async ({ browser, video, _artifactsDir }, use, testInfo) => {
+  _contextFactory: [async ({ browser, video, _artifactsDir }, use, testInfo) => {
     let videoMode = typeof video === 'string' ? video : video.mode;
     if (videoMode === 'retry-with-video')
       videoMode = 'on-first-retry';
@@ -476,7 +487,7 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
 
     if (prependToError)
       testInfo.errors.push({ message: prependToError });
-  },
+  }, { scope: 'test',  _title: 'context' } as any],
 
   context: async ({ _contextFactory }, use) => {
     await use(await _contextFactory());

packages/playwright-test/src/timeoutManager.ts

@@ -31,7 +31,7 @@ type RunnableDescription = {
 };
 
 export type FixtureDescription = {
-  fixture: string;
+  title: string;
   location?: Location;
   slot?: TimeSlot;  // Falls back to current runnable slot.
 };
@@ -123,7 +123,9 @@ export class TimeoutManager {
     }
     const fixtureWithSlot = this._fixture?.slot ? this._fixture : undefined;
     if (fixtureWithSlot)
-      suffix = ` in fixture "${fixtureWithSlot.fixture}"`;
+      suffix = ` by ${fixtureWithSlot.title}`;
+    else if (this._fixture)
+      suffix = ` while running ${this._fixture.title}`;
     const message = colors.red(`Timeout of ${this._currentSlot().timeout}ms exceeded${suffix}.`);
     const location = (fixtureWithSlot || this._runnable).location;
     return {

tests/playwright-test/fixture-errors.spec.ts

@@ -471,10 +471,10 @@ test('should not report fixture teardown timeout twice', async ({ runInlineTest
   }, { reporter: 'list', timeout: 1000 });
   expect(result.exitCode).toBe(1);
   expect(result.failed).toBe(1);
-  expect(result.output).toContain('in fixtures teardown');
+  expect(result.output).toContain('Timeout of 1000ms exceeded while running fixture "fixture" teardown.');
   expect(stripAnsi(result.output)).not.toContain('pwt.test.extend'); // Should not point to the location.
-  // TODO: this should be "1" actually.
+  // TODO: this should be "not.toContain" actually.
-  expect(countTimes(result.output, 'in fixtures teardown')).not.toBe(1);
+  expect(result.output).toContain('in fixtures teardown');
 });
 
 test('should handle fixture teardown error after test timeout and continue', async ({ runInlineTest }) => {
@@ -529,6 +529,6 @@ test('should report worker fixture teardown with debug info', async ({ runInline
     'a.spec.ts:12:9 › good18',
     'a.spec.ts:12:9 › good19',
     '',
-    'Timeout of 1000ms exceeded in fixtures teardown.',
+    'Timeout of 1000ms exceeded while running fixture "fixture" teardown.',
   ].join('\n'));
 });

tests/playwright-test/playwright.connect.spec.ts

@@ -0,0 +1,97 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should work with connectOptions', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = {
+        globalSetup: './global-setup',
+        use: {
+          connectOptions: {
+            wsEndpoint: process.env.CONNECT_WS_ENDPOINT,
+          },
+        },
+      };
+    `,
+    'global-setup.ts': `
+      module.exports = async () => {
+        const server = await pwt.chromium.launchServer();
+        process.env.CONNECT_WS_ENDPOINT = server.wsEndpoint();
+        return () => server.close();
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test.use({ locale: 'fr-CH' });
+      test('pass', async ({ page }) => {
+        await page.setContent('<div>PASS</div>');
+        await expect(page.locator('div')).toHaveText('PASS');
+        expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should throw with bad connectOptions', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = {
+        use: {
+          connectOptions: {
+            wsEndpoint: 'http://does-not-exist-bad-domain.oh-no-should-not-work',
+          },
+        },
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+        await page.setContent('<div>PASS</div>');
+        await expect(page.locator('div')).toHaveText('PASS');
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.output).toContain('browserType.connect:');
+});
+
+test('should respect connectOptions.timeout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = {
+        use: {
+          connectOptions: {
+            wsEndpoint: 'wss://locahost:5678',
+            timeout: 1,
+          },
+        },
+      };
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('pass', async ({ page }) => {
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.output).toContain('browserType.connect: Timeout 1ms exceeded.');
+});

tests/playwright-test/playwright.spec.ts

@@ -563,83 +563,3 @@ test('should work with video.path() throwing', async ({ runInlineTest }, testInf
   const video = fs.readdirSync(dir).find(file => file.endsWith('webm'));
   expect(video).toBeTruthy();
 });
-
-test('should work with connectOptions', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.js': `
-      module.exports = {
-        globalSetup: './global-setup',
-        use: {
-          connectOptions: {
-            wsEndpoint: process.env.CONNECT_WS_ENDPOINT,
-          },
-        },
-      };
-    `,
-    'global-setup.ts': `
-      module.exports = async () => {
-        const server = await pwt.chromium.launchServer();
-        process.env.CONNECT_WS_ENDPOINT = server.wsEndpoint();
-        return () => server.close();
-      };
-    `,
-    'a.test.ts': `
-      const { test } = pwt;
-      test.use({ locale: 'fr-CH' });
-      test('pass', async ({ page }) => {
-        await page.setContent('<div>PASS</div>');
-        await expect(page.locator('div')).toHaveText('PASS');
-        expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
-      });
-    `,
-  });
-  expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(1);
-});
-
-test('should throw with bad connectOptions', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.js': `
-      module.exports = {
-        use: {
-          connectOptions: {
-            wsEndpoint: 'http://does-not-exist-bad-domain.oh-no-should-not-work',
-          },
-        },
-      };
-    `,
-    'a.test.ts': `
-      const { test } = pwt;
-      test('pass', async ({ page }) => {
-        await page.setContent('<div>PASS</div>');
-        await expect(page.locator('div')).toHaveText('PASS');
-      });
-    `,
-  });
-  expect(result.exitCode).toBe(1);
-  expect(result.passed).toBe(0);
-  expect(result.output).toContain('browserType.connect:');
-});
-
-test('should respect connectOptions.timeout', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.js': `
-      module.exports = {
-        use: {
-          connectOptions: {
-            wsEndpoint: 'wss://locahost:5678',
-            timeout: 1,
-          },
-        },
-      };
-    `,
-    'a.test.ts': `
-      const { test } = pwt;
-      test('pass', async ({ page }) => {
-      });
-    `,
-  });
-  expect(result.exitCode).toBe(1);
-  expect(result.passed).toBe(0);
-  expect(result.output).toContain('browserType.connect: Timeout 1ms exceeded.');
-});

tests/playwright-test/timeout.spec.ts

@@ -178,7 +178,7 @@ test('should respect fixture timeout', async ({ runInlineTest }) => {
         slowSetup: [async ({}, use) => {
           await new Promise(f => setTimeout(f, 2000));
           await use('hey');
-        }, { timeout: 500 }],
+        }, { timeout: 500, _title: 'custom title' }],
         slowTeardown: [async ({}, use) => {
           await use('hey');
           await new Promise(f => setTimeout(f, 2000));
@@ -196,8 +196,8 @@ test('should respect fixture timeout', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(1);
   expect(result.failed).toBe(2);
-  expect(result.output).toContain('Timeout of 500ms exceeded in fixture "slowSetup"');
+  expect(result.output).toContain('Timeout of 500ms exceeded by fixture "custom title" setup.');
-  expect(result.output).toContain('Timeout of 400ms exceeded in fixture "slowTeardown"');
+  expect(result.output).toContain('Timeout of 400ms exceeded by fixture "slowTeardown" teardown.');
   expect(stripAnsi(result.output)).toContain('> 5 |       const test = pwt.test.extend({');
 });
 
@@ -222,7 +222,7 @@ test('should respect test.setTimeout in the worker fixture', async ({ runInlineT
         slowTeardown: [async ({}, use) => {
           await use('hey');
           await new Promise(f => setTimeout(f, 2000));
-        }, { scope: 'worker', timeout: 400 }],
+        }, { scope: 'worker', timeout: 400, _title: 'custom title' }],
       });
       test('test ok', async ({ fixture, noTimeout }) => {
         await new Promise(f => setTimeout(f, 1000));
@@ -236,8 +236,8 @@ test('should respect test.setTimeout in the worker fixture', async ({ runInlineT
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(2);
   expect(result.failed).toBe(1);
-  expect(result.output).toContain('Timeout of 500ms exceeded in fixture "slowSetup"');
+  expect(result.output).toContain('Timeout of 500ms exceeded by fixture "slowSetup" setup.');
-  expect(result.output).toContain('Timeout of 400ms exceeded in fixture "slowTeardown"');
+  expect(result.output).toContain('Timeout of 400ms exceeded by fixture "custom title" teardown.');
 });
 
 test('fixture time in beforeAll hook should not affect test', async ({ runInlineTest }) => {