feat(test): more information about timing out fixtures (#13546)
- Always show a fixture that was running during timeout. - Give custom titles to built-in fixtures. - Specify setup/teardown fixture phase in the message. - Split connect vs launch browser fixtures for better naming. Example timeout message: ```log Timeout of 2000ms exceeded while running fixture "built-in playwright configuration" teardown. ```
This commit is contained in:
Dmitry Gozman
GitHub
parent
e2fff31848
commit
166675b9c1
|
|
@ -31,6 +31,7 @@ type FixtureRegistration = {
|
|||
fn: Function | any; // Either a fixture function, or a fixture value.
|
||||
auto: boolean;
|
||||
option: boolean;
|
||||
customTitle?: string;
|
||||
timeout?: number;
|
||||
deps: string[]; // Names of the dependencies, ({ foo, bar }) => {...}
|
||||
id: string; // Unique id, to differentiate between fixtures with the same name.
|
||||
|
|
@ -54,7 +55,7 @@ class Fixture {
|
|||
this.usages = new Set();
|
||||
this.value = null;
|
||||
this._runnableDescription = {
|
||||
fixture: this.registration.name,
|
||||
title: `fixture "${this.registration.customTitle || this.registration.name}" setup`,
|
||||
location: registration.location,
|
||||
slot: this.registration.timeout === undefined ? undefined : {
|
||||
timeout: this.registration.timeout,
|
||||
|
|
@ -117,6 +118,7 @@ class Fixture {
|
|||
this.usages.clear();
|
||||
if (this._useFuncFinished) {
|
||||
debugTest(`teardown ${this.registration.name}`);
|
||||
this._runnableDescription.title = `fixture "${this.registration.customTitle || this.registration.name}" teardown`;
|
||||
timeoutManager.setCurrentFixture(this._runnableDescription);
|
||||
this._useFuncFinished.resolve();
|
||||
await this._selfTeardownComplete;
|
||||
|
|
@ -147,13 +149,14 @@ export class FixturePool {
|
|||
for (const entry of Object.entries(fixtures)) {
|
||||
const name = entry[0];
|
||||
let value = entry[1];
|
||||
let options: { auto: boolean, scope: FixtureScope, option: boolean, timeout: number | undefined } | undefined;
|
||||
let options: { auto: boolean, scope: FixtureScope, option: boolean, timeout: number | undefined, customTitle: string | undefined } | undefined;
|
||||
if (isFixtureTuple(value)) {
|
||||
options = {
|
||||
auto: !!value[1].auto,
|
||||
scope: value[1].scope || 'test',
|
||||
option: !!value[1].option,
|
||||
timeout: value[1].timeout,
|
||||
customTitle: (value[1] as any)._title,
|
||||
};
|
||||
value = value[0];
|
||||
}
|
||||
|
|
@ -166,9 +169,9 @@ export class FixturePool {
|
|||
if (previous.auto !== options.auto)
|
||||
throw errorWithLocations(`Fixture "${name}" has already been registered as a { auto: '${previous.scope}' } fixture.`, { location, name }, previous);
|
||||
} else if (previous) {
|
||||
options = { auto: previous.auto, scope: previous.scope, option: previous.option, timeout: previous.timeout };
|
||||
options = { auto: previous.auto, scope: previous.scope, option: previous.option, timeout: previous.timeout, customTitle: previous.customTitle };
|
||||
} else if (!options) {
|
||||
options = { auto: false, scope: 'test', option: false, timeout: undefined };
|
||||
options = { auto: false, scope: 'test', option: false, timeout: undefined, customTitle: undefined };
|
||||
}
|
||||
|
||||
if (options.scope !== 'test' && options.scope !== 'worker')
|
||||
|
|
@ -177,7 +180,7 @@ export class FixturePool {
|
|||
throw errorWithLocations(`Cannot use({ ${name} }) in a describe group, because it forces a new worker.\nMake it top-level in the test file or put in the configuration file.`, { location, name });
|
||||
|
||||
const deps = fixtureParameterNames(fn, location);
|
||||
const registration: FixtureRegistration = { id: '', name, location, scope: options.scope, fn, auto: options.auto, option: options.option, timeout: options.timeout, deps, super: previous };
|
||||
const registration: FixtureRegistration = { id: '', name, location, scope: options.scope, fn, auto: options.auto, option: options.option, timeout: options.timeout, customTitle: options.customTitle, deps, super: previous };
|
||||
registrationId(registration);
|
||||
this.registrations.set(name, registration);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext, Video, APIRequestContext, Tracing } from 'playwright-core';
|
||||
import type { LaunchOptions, BrowserContextOptions, Page, Browser, BrowserContext, Video, APIRequestContext, Tracing } from 'playwright-core';
|
||||
import type { TestType, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, TestInfo } from '../types/test';
|
||||
import { rootTestType } from './testType';
|
||||
import { createGuid, debugMode } from 'playwright-core/lib/utils';
|
||||
|
|
@ -44,6 +44,7 @@ type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & {
|
|||
_contextFactory: (options?: BrowserContextOptions) => Promise<BrowserContext>;
|
||||
};
|
||||
type WorkerFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & {
|
||||
_connectedBrowser: Browser | undefined,
|
||||
_browserOptions: LaunchOptions;
|
||||
_artifactsDir: () => string;
|
||||
_snapshotSuffix: string;
|
||||
|
|
@ -86,12 +87,12 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
|
|||
});
|
||||
if (dir)
|
||||
await removeFolders([dir]);
|
||||
}, { scope: 'worker' }],
|
||||
}, { scope: 'worker', _title: 'built-in playwright configuration' } as any],
|
||||
|
||||
_browserOptions: [async ({ playwright, headless, channel, launchOptions }, use) => {
|
||||
const options: LaunchOptions = {
|
||||
handleSIGINT: false,
|
||||
timeout: 30000, // 30 seconds
|
||||
timeout: 0,
|
||||
...launchOptions,
|
||||
};
|
||||
if (headless !== undefined)
|
||||
|
|
@ -106,27 +107,37 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
|
|||
(browserType as any)._defaultLaunchOptions = undefined;
|
||||
}, { scope: 'worker', auto: true }],
|
||||
|
||||
browser: [async ({ playwright, browserName, channel, headless, connectOptions }, use) => {
|
||||
_connectedBrowser: [async ({ playwright, browserName, channel, headless, connectOptions }, use) => {
|
||||
if (!connectOptions) {
|
||||
await use(undefined);
|
||||
return;
|
||||
}
|
||||
if (!['chromium', 'firefox', 'webkit'].includes(browserName))
|
||||
throw new Error(`Unexpected browserName "${browserName}", must be one of "chromium", "firefox" or "webkit"`);
|
||||
if (connectOptions) {
|
||||
const browser = await playwright[browserName].connect(connectOptions.wsEndpoint, {
|
||||
headers: {
|
||||
'x-playwright-browser': channel || browserName,
|
||||
'x-playwright-headless': headless ? '1' : '0',
|
||||
...connectOptions.headers,
|
||||
},
|
||||
timeout: connectOptions.timeout ?? 3 * 60 * 1000, // 3 minutes
|
||||
});
|
||||
await use(browser);
|
||||
await browser.close();
|
||||
const browser = await playwright[browserName].connect(connectOptions.wsEndpoint, {
|
||||
headers: {
|
||||
'x-playwright-browser': channel || browserName,
|
||||
'x-playwright-headless': headless ? '1' : '0',
|
||||
...connectOptions.headers,
|
||||
},
|
||||
timeout: connectOptions.timeout ?? 3 * 60 * 1000, // 3 minutes
|
||||
});
|
||||
await use(browser);
|
||||
await browser.close();
|
||||
}, { scope: 'worker', timeout: 0, _title: 'remote connection' } as any],
|
||||
|
||||
browser: [async ({ playwright, browserName, _connectedBrowser }, use) => {
|
||||
if (_connectedBrowser) {
|
||||
await use(_connectedBrowser);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!['chromium', 'firefox', 'webkit'].includes(browserName))
|
||||
throw new Error(`Unexpected browserName "${browserName}", must be one of "chromium", "firefox" or "webkit"`);
|
||||
const browser = await playwright[browserName].launch();
|
||||
await use(browser);
|
||||
await browser.close();
|
||||
}, { scope: 'worker', timeout: 0 } ],
|
||||
}, { scope: 'worker' } ],
|
||||
|
||||
acceptDownloads: [ undefined, { option: true } ],
|
||||
bypassCSP: [ undefined, { option: true } ],
|
||||
|
|
@ -422,9 +433,9 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
|
|||
else
|
||||
await fs.promises.unlink(file).catch(() => {});
|
||||
}));
|
||||
}, { auto: true }],
|
||||
}, { auto: true, _title: 'built-in playwright configuration' } as any],
|
||||
|
||||
_contextFactory: async ({ browser, video, _artifactsDir }, use, testInfo) => {
|
||||
_contextFactory: [async ({ browser, video, _artifactsDir }, use, testInfo) => {
|
||||
let videoMode = typeof video === 'string' ? video : video.mode;
|
||||
if (videoMode === 'retry-with-video')
|
||||
videoMode = 'on-first-retry';
|
||||
|
|
@ -476,7 +487,7 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
|
|||
|
||||
if (prependToError)
|
||||
testInfo.errors.push({ message: prependToError });
|
||||
},
|
||||
}, { scope: 'test', _title: 'context' } as any],
|
||||
|
||||
context: async ({ _contextFactory }, use) => {
|
||||
await use(await _contextFactory());
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ type RunnableDescription = {
|
|||
};
|
||||
|
||||
export type FixtureDescription = {
|
||||
fixture: string;
|
||||
title: string;
|
||||
location?: Location;
|
||||
slot?: TimeSlot; // Falls back to current runnable slot.
|
||||
};
|
||||
|
|
@ -123,7 +123,9 @@ export class TimeoutManager {
|
|||
}
|
||||
const fixtureWithSlot = this._fixture?.slot ? this._fixture : undefined;
|
||||
if (fixtureWithSlot)
|
||||
suffix = ` in fixture "${fixtureWithSlot.fixture}"`;
|
||||
suffix = ` by ${fixtureWithSlot.title}`;
|
||||
else if (this._fixture)
|
||||
suffix = ` while running ${this._fixture.title}`;
|
||||
const message = colors.red(`Timeout of ${this._currentSlot().timeout}ms exceeded${suffix}.`);
|
||||
const location = (fixtureWithSlot || this._runnable).location;
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -471,10 +471,10 @@ test('should not report fixture teardown timeout twice', async ({ runInlineTest
|
|||
}, { reporter: 'list', timeout: 1000 });
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.output).toContain('in fixtures teardown');
|
||||
expect(result.output).toContain('Timeout of 1000ms exceeded while running fixture "fixture" teardown.');
|
||||
expect(stripAnsi(result.output)).not.toContain('pwt.test.extend'); // Should not point to the location.
|
||||
// TODO: this should be "1" actually.
|
||||
expect(countTimes(result.output, 'in fixtures teardown')).not.toBe(1);
|
||||
// TODO: this should be "not.toContain" actually.
|
||||
expect(result.output).toContain('in fixtures teardown');
|
||||
});
|
||||
|
||||
test('should handle fixture teardown error after test timeout and continue', async ({ runInlineTest }) => {
|
||||
|
|
@ -529,6 +529,6 @@ test('should report worker fixture teardown with debug info', async ({ runInline
|
|||
'a.spec.ts:12:9 › good18',
|
||||
'a.spec.ts:12:9 › good19',
|
||||
'',
|
||||
'Timeout of 1000ms exceeded in fixtures teardown.',
|
||||
'Timeout of 1000ms exceeded while running fixture "fixture" teardown.',
|
||||
].join('\n'));
|
||||
});
|
||||
|
|
|
|||
97
tests/playwright-test/playwright.connect.spec.ts
Normal file
97
tests/playwright-test/playwright.connect.spec.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { test, expect } from './playwright-test-fixtures';
|
||||
|
||||
test('should work with connectOptions', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.js': `
|
||||
module.exports = {
|
||||
globalSetup: './global-setup',
|
||||
use: {
|
||||
connectOptions: {
|
||||
wsEndpoint: process.env.CONNECT_WS_ENDPOINT,
|
||||
},
|
||||
},
|
||||
};
|
||||
`,
|
||||
'global-setup.ts': `
|
||||
module.exports = async () => {
|
||||
const server = await pwt.chromium.launchServer();
|
||||
process.env.CONNECT_WS_ENDPOINT = server.wsEndpoint();
|
||||
return () => server.close();
|
||||
};
|
||||
`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test.use({ locale: 'fr-CH' });
|
||||
test('pass', async ({ page }) => {
|
||||
await page.setContent('<div>PASS</div>');
|
||||
await expect(page.locator('div')).toHaveText('PASS');
|
||||
expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
|
||||
});
|
||||
`,
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
});
|
||||
|
||||
test('should throw with bad connectOptions', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.js': `
|
||||
module.exports = {
|
||||
use: {
|
||||
connectOptions: {
|
||||
wsEndpoint: 'http://does-not-exist-bad-domain.oh-no-should-not-work',
|
||||
},
|
||||
},
|
||||
};
|
||||
`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test('pass', async ({ page }) => {
|
||||
await page.setContent('<div>PASS</div>');
|
||||
await expect(page.locator('div')).toHaveText('PASS');
|
||||
});
|
||||
`,
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.passed).toBe(0);
|
||||
expect(result.output).toContain('browserType.connect:');
|
||||
});
|
||||
|
||||
test('should respect connectOptions.timeout', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.js': `
|
||||
module.exports = {
|
||||
use: {
|
||||
connectOptions: {
|
||||
wsEndpoint: 'wss://locahost:5678',
|
||||
timeout: 1,
|
||||
},
|
||||
},
|
||||
};
|
||||
`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test('pass', async ({ page }) => {
|
||||
});
|
||||
`,
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.passed).toBe(0);
|
||||
expect(result.output).toContain('browserType.connect: Timeout 1ms exceeded.');
|
||||
});
|
||||
|
|
@ -563,83 +563,3 @@ test('should work with video.path() throwing', async ({ runInlineTest }, testInf
|
|||
const video = fs.readdirSync(dir).find(file => file.endsWith('webm'));
|
||||
expect(video).toBeTruthy();
|
||||
});
|
||||
|
||||
test('should work with connectOptions', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.js': `
|
||||
module.exports = {
|
||||
globalSetup: './global-setup',
|
||||
use: {
|
||||
connectOptions: {
|
||||
wsEndpoint: process.env.CONNECT_WS_ENDPOINT,
|
||||
},
|
||||
},
|
||||
};
|
||||
`,
|
||||
'global-setup.ts': `
|
||||
module.exports = async () => {
|
||||
const server = await pwt.chromium.launchServer();
|
||||
process.env.CONNECT_WS_ENDPOINT = server.wsEndpoint();
|
||||
return () => server.close();
|
||||
};
|
||||
`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test.use({ locale: 'fr-CH' });
|
||||
test('pass', async ({ page }) => {
|
||||
await page.setContent('<div>PASS</div>');
|
||||
await expect(page.locator('div')).toHaveText('PASS');
|
||||
expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
|
||||
});
|
||||
`,
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
});
|
||||
|
||||
test('should throw with bad connectOptions', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.js': `
|
||||
module.exports = {
|
||||
use: {
|
||||
connectOptions: {
|
||||
wsEndpoint: 'http://does-not-exist-bad-domain.oh-no-should-not-work',
|
||||
},
|
||||
},
|
||||
};
|
||||
`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test('pass', async ({ page }) => {
|
||||
await page.setContent('<div>PASS</div>');
|
||||
await expect(page.locator('div')).toHaveText('PASS');
|
||||
});
|
||||
`,
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.passed).toBe(0);
|
||||
expect(result.output).toContain('browserType.connect:');
|
||||
});
|
||||
|
||||
test('should respect connectOptions.timeout', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.js': `
|
||||
module.exports = {
|
||||
use: {
|
||||
connectOptions: {
|
||||
wsEndpoint: 'wss://locahost:5678',
|
||||
timeout: 1,
|
||||
},
|
||||
},
|
||||
};
|
||||
`,
|
||||
'a.test.ts': `
|
||||
const { test } = pwt;
|
||||
test('pass', async ({ page }) => {
|
||||
});
|
||||
`,
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.passed).toBe(0);
|
||||
expect(result.output).toContain('browserType.connect: Timeout 1ms exceeded.');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ test('should respect fixture timeout', async ({ runInlineTest }) => {
|
|||
slowSetup: [async ({}, use) => {
|
||||
await new Promise(f => setTimeout(f, 2000));
|
||||
await use('hey');
|
||||
}, { timeout: 500 }],
|
||||
}, { timeout: 500, _title: 'custom title' }],
|
||||
slowTeardown: [async ({}, use) => {
|
||||
await use('hey');
|
||||
await new Promise(f => setTimeout(f, 2000));
|
||||
|
|
@ -196,8 +196,8 @@ test('should respect fixture timeout', async ({ runInlineTest }) => {
|
|||
expect(result.exitCode).toBe(1);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.failed).toBe(2);
|
||||
expect(result.output).toContain('Timeout of 500ms exceeded in fixture "slowSetup"');
|
||||
expect(result.output).toContain('Timeout of 400ms exceeded in fixture "slowTeardown"');
|
||||
expect(result.output).toContain('Timeout of 500ms exceeded by fixture "custom title" setup.');
|
||||
expect(result.output).toContain('Timeout of 400ms exceeded by fixture "slowTeardown" teardown.');
|
||||
expect(stripAnsi(result.output)).toContain('> 5 | const test = pwt.test.extend({');
|
||||
});
|
||||
|
||||
|
|
@ -222,7 +222,7 @@ test('should respect test.setTimeout in the worker fixture', async ({ runInlineT
|
|||
slowTeardown: [async ({}, use) => {
|
||||
await use('hey');
|
||||
await new Promise(f => setTimeout(f, 2000));
|
||||
}, { scope: 'worker', timeout: 400 }],
|
||||
}, { scope: 'worker', timeout: 400, _title: 'custom title' }],
|
||||
});
|
||||
test('test ok', async ({ fixture, noTimeout }) => {
|
||||
await new Promise(f => setTimeout(f, 1000));
|
||||
|
|
@ -236,8 +236,8 @@ test('should respect test.setTimeout in the worker fixture', async ({ runInlineT
|
|||
expect(result.exitCode).toBe(1);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.output).toContain('Timeout of 500ms exceeded in fixture "slowSetup"');
|
||||
expect(result.output).toContain('Timeout of 400ms exceeded in fixture "slowTeardown"');
|
||||
expect(result.output).toContain('Timeout of 500ms exceeded by fixture "slowSetup" setup.');
|
||||
expect(result.output).toContain('Timeout of 400ms exceeded by fixture "custom title" teardown.');
|
||||
});
|
||||
|
||||
test('fixture time in beforeAll hook should not affect test', async ({ runInlineTest }) => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue