feat(fixtures): per-fixture timeout (#12751)

By default, fixtures share timeout with the test they are instantiated for.
However, for more heavy fixtures, especially worker-scoped ones, it makes
sense to have a separate timeout.

This introduces `{ timeout: number }` option to the list of fixture options
that opts the fixture into a dedicated timeout rather than sharing it
with the test.
This commit is contained in:
Dmitry Gozman 2022-03-17 09:36:03 -07:00 committed by GitHub
parent b3ca805591
commit 25483452c0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 476 additions and 142 deletions

View file

@ -880,7 +880,7 @@ Test function that takes one or two arguments: an object with fixtures and optio
## method: Test.setTimeout ## method: Test.setTimeout
Changes the timeout for the test. Learn more about [various timeouts](./test-timeouts.md). Changes the timeout for the test. Zero means no timeout. Learn more about [various timeouts](./test-timeouts.md).
```js js-flavor=js ```js js-flavor=js
const { test, expect } = require('@playwright/test'); const { test, expect } = require('@playwright/test');

View file

@ -466,6 +466,41 @@ export const test = base.extend<{ saveLogs: void }>({
export { expect } from '@playwright/test'; export { expect } from '@playwright/test';
``` ```
## Fixture timeout
By default, fixture shares timeout with the test. However, for slow fixtures, especially [worker-scoped](#worker-scoped-fixtures) ones, it is convenient to have a separate timeout. This way you can keep the overall test timeout small, and give the slow fixture more time.
```js js-flavor=js
const { test: base, expect } = require('@playwright/test');
const test = base.extend({
slowFixture: [async ({}, use) => {
// ... perform a slow operation ...
await use('hello');
}, { timeout: 60000 }]
});
test('example test', async ({ slowFixture }) => {
// ...
});
```
```js js-flavor=ts
import { test as base, expect } from '@playwright/test';
const test = base.extend<{ slowFixture: string }>({
slowFixture: [async ({}, use) => {
// ... perform a slow operation ...
await use('hello');
}, { timeout: 60000 }]
});
test('example test', async ({ slowFixture }) => {
// ...
});
```
## Fixtures-options ## Fixtures-options
:::note :::note

View file

@ -16,6 +16,7 @@ Playwright Test has multiple configurable timeouts for various tasks.
|Action timeout| no timeout |Timeout for each action:<br/><span style={{textTransform:'uppercase',fontSize:'smaller',fontWeight:'bold',opacity:'0.6'}}>Set default</span><br/><code>{`config = { use: { actionTimeout: 10000 } }`}</code><br/><span style={{textTransform: 'uppercase',fontSize: 'smaller', fontWeight: 'bold', opacity: '0.6'}}>Override</span><br/><code>{`locator.click({ timeout: 10000 })`}</code>| |Action timeout| no timeout |Timeout for each action:<br/><span style={{textTransform:'uppercase',fontSize:'smaller',fontWeight:'bold',opacity:'0.6'}}>Set default</span><br/><code>{`config = { use: { actionTimeout: 10000 } }`}</code><br/><span style={{textTransform: 'uppercase',fontSize: 'smaller', fontWeight: 'bold', opacity: '0.6'}}>Override</span><br/><code>{`locator.click({ timeout: 10000 })`}</code>|
|Navigation timeout| no timeout |Timeout for each navigation action:<br/><span style={{textTransform:'uppercase',fontSize:'smaller',fontWeight:'bold',opacity:'0.6'}}>Set default</span><br/><code>{`config = { use: { navigationTimeout: 30000 } }`}</code><br/><span style={{textTransform: 'uppercase',fontSize: 'smaller', fontWeight: 'bold', opacity: '0.6'}}>Override</span><br/><code>{`page.goto('/', { timeout: 30000 })`}</code>| |Navigation timeout| no timeout |Timeout for each navigation action:<br/><span style={{textTransform:'uppercase',fontSize:'smaller',fontWeight:'bold',opacity:'0.6'}}>Set default</span><br/><code>{`config = { use: { navigationTimeout: 30000 } }`}</code><br/><span style={{textTransform: 'uppercase',fontSize: 'smaller', fontWeight: 'bold', opacity: '0.6'}}>Override</span><br/><code>{`page.goto('/', { timeout: 30000 })`}</code>|
|Global timeout|no timeout |Global timeout for the whole test run:<br/><span style={{textTransform:'uppercase',fontSize:'smaller',fontWeight:'bold',opacity:'0.6'}}>Set in config</span><br/><code>{`config = { globalTimeout: 60*60*1000 }`}</code><br/>| |Global timeout|no timeout |Global timeout for the whole test run:<br/><span style={{textTransform:'uppercase',fontSize:'smaller',fontWeight:'bold',opacity:'0.6'}}>Set in config</span><br/><code>{`config = { globalTimeout: 60*60*1000 }`}</code><br/>|
|Fixture timeout|no timeout |Timeout for an individual fixture:<br/><span style={{textTransform:'uppercase',fontSize:'smaller',fontWeight:'bold',opacity:'0.6'}}>Set in fixture</span><br/><code>{`{ scope: 'test', timeout: 30000 }`}</code><br/>|
## Test timeout ## Test timeout
@ -89,7 +90,7 @@ test('very slow test', async ({ page }) => {
API reference: [`method: Test.setTimeout`] and [`method: Test.slow`]. API reference: [`method: Test.setTimeout`] and [`method: Test.slow`].
### Change timeout from a hook or fixture ### Change timeout from a hook
```js js-flavor=js ```js js-flavor=js
const { test, expect } = require('@playwright/test'); const { test, expect } = require('@playwright/test');
@ -279,3 +280,39 @@ export default config;
``` ```
API reference: [`property: TestConfig.globalTimeout`]. API reference: [`property: TestConfig.globalTimeout`].
## Fixture timeout
By default, [fixture](./test-fixtures) shares timeout with the test. However, for slow fixtures, especially [worker-scoped](./test-fixtures#worker-scoped-fixtures) ones, it is convenient to have a separate timeout. This way you can keep the overall test timeout small, and give the slow fixture more time.
```js js-flavor=js
const { test: base, expect } = require('@playwright/test');
const test = base.extend({
slowFixture: [async ({}, use) => {
// ... perform a slow operation ...
await use('hello');
}, { timeout: 60000 }]
});
test('example test', async ({ slowFixture }) => {
// ...
});
```
```js js-flavor=ts
import { test as base, expect } from '@playwright/test';
const test = base.extend<{ slowFixture: string }>({
slowFixture: [async ({}, use) => {
// ... perform a slow operation ...
await use('hello');
}, { timeout: 60000 }]
});
test('example test', async ({ slowFixture }) => {
// ...
});
```
API reference: [`method: Test.extend`].

View file

@ -16,11 +16,13 @@
import { formatLocation, debugTest } from './util'; import { formatLocation, debugTest } from './util';
import * as crypto from 'crypto'; import * as crypto from 'crypto';
import { FixturesWithLocation, Location, WorkerInfo, TestInfo } from './types'; import { FixturesWithLocation, Location, WorkerInfo } from './types';
import { ManualPromise } from 'playwright-core/lib/utils/async'; import { ManualPromise } from 'playwright-core/lib/utils/async';
import { TestInfoImpl } from './testInfo';
import { FixtureDescription, TimeoutManager } from './timeoutManager';
type FixtureScope = 'test' | 'worker'; type FixtureScope = 'test' | 'worker';
type FixtureOptions = { auto?: boolean, scope?: FixtureScope, option?: boolean }; type FixtureOptions = { auto?: boolean, scope?: FixtureScope, option?: boolean, timeout?: number | undefined };
type FixtureTuple = [ value: any, options: FixtureOptions ]; type FixtureTuple = [ value: any, options: FixtureOptions ];
type FixtureRegistration = { type FixtureRegistration = {
location: Location; // Fixutre registration location. location: Location; // Fixutre registration location.
@ -29,6 +31,7 @@ type FixtureRegistration = {
fn: Function | any; // Either a fixture function, or a fixture value. fn: Function | any; // Either a fixture function, or a fixture value.
auto: boolean; auto: boolean;
option: boolean; option: boolean;
timeout?: number;
deps: string[]; // Names of the dependencies, ({ foo, bar }) => {...} deps: string[]; // Names of the dependencies, ({ foo, bar }) => {...}
id: string; // Unique id, to differentiate between fixtures with the same name. id: string; // Unique id, to differentiate between fixtures with the same name.
super?: FixtureRegistration; // A fixture override can use the previous version of the fixture. super?: FixtureRegistration; // A fixture override can use the previous version of the fixture.
@ -43,15 +46,24 @@ class Fixture {
_useFuncFinished: ManualPromise<void> | undefined; _useFuncFinished: ManualPromise<void> | undefined;
_selfTeardownComplete: Promise<void> | undefined; _selfTeardownComplete: Promise<void> | undefined;
_teardownWithDepsComplete: Promise<void> | undefined; _teardownWithDepsComplete: Promise<void> | undefined;
_runnableDescription: FixtureDescription;
constructor(runner: FixtureRunner, registration: FixtureRegistration) { constructor(runner: FixtureRunner, registration: FixtureRegistration) {
this.runner = runner; this.runner = runner;
this.registration = registration; this.registration = registration;
this.usages = new Set(); this.usages = new Set();
this.value = null; this.value = null;
this._runnableDescription = {
fixture: this.registration.name,
location: registration.location,
slot: this.registration.timeout === undefined ? undefined : {
timeout: this.registration.timeout,
elapsed: 0,
}
};
} }
async setup(testInfo: TestInfo) { async setup(testInfo: TestInfoImpl) {
if (typeof this.registration.fn !== 'function') { if (typeof this.registration.fn !== 'function') {
this.value = this.registration.fn; this.value = this.registration.fn;
return; return;
@ -79,6 +91,7 @@ class Fixture {
}; };
const workerInfo: WorkerInfo = { config: testInfo.config, parallelIndex: testInfo.parallelIndex, workerIndex: testInfo.workerIndex, project: testInfo.project }; const workerInfo: WorkerInfo = { config: testInfo.config, parallelIndex: testInfo.parallelIndex, workerIndex: testInfo.workerIndex, project: testInfo.project };
const info = this.registration.scope === 'worker' ? workerInfo : testInfo; const info = this.registration.scope === 'worker' ? workerInfo : testInfo;
testInfo._timeoutManager.setCurrentFixture(this._runnableDescription);
this._selfTeardownComplete = Promise.resolve().then(() => this.registration.fn(params, useFunc, info)).catch((e: any) => { this._selfTeardownComplete = Promise.resolve().then(() => this.registration.fn(params, useFunc, info)).catch((e: any) => {
if (!useFuncStarted.isDone()) if (!useFuncStarted.isDone())
useFuncStarted.reject(e); useFuncStarted.reject(e);
@ -86,25 +99,28 @@ class Fixture {
throw e; throw e;
}); });
await useFuncStarted; await useFuncStarted;
testInfo._timeoutManager.setCurrentFixture(undefined);
} }
async teardown() { async teardown(timeoutManager: TimeoutManager) {
if (!this._teardownWithDepsComplete) if (!this._teardownWithDepsComplete)
this._teardownWithDepsComplete = this._teardownInternal(); this._teardownWithDepsComplete = this._teardownInternal(timeoutManager);
await this._teardownWithDepsComplete; await this._teardownWithDepsComplete;
} }
private async _teardownInternal() { private async _teardownInternal(timeoutManager: TimeoutManager) {
if (typeof this.registration.fn !== 'function') if (typeof this.registration.fn !== 'function')
return; return;
try { try {
for (const fixture of this.usages) for (const fixture of this.usages)
await fixture.teardown(); await fixture.teardown(timeoutManager);
this.usages.clear(); this.usages.clear();
if (this._useFuncFinished) { if (this._useFuncFinished) {
debugTest(`teardown ${this.registration.name}`); debugTest(`teardown ${this.registration.name}`);
timeoutManager.setCurrentFixture(this._runnableDescription);
this._useFuncFinished.resolve(); this._useFuncFinished.resolve();
await this._selfTeardownComplete; await this._selfTeardownComplete;
timeoutManager.setCurrentFixture(undefined);
} }
} finally { } finally {
this.runner.instanceForId.delete(this.registration.id); this.runner.instanceForId.delete(this.registration.id);
@ -113,7 +129,7 @@ class Fixture {
} }
function isFixtureTuple(value: any): value is FixtureTuple { function isFixtureTuple(value: any): value is FixtureTuple {
return Array.isArray(value) && typeof value[1] === 'object' && ('scope' in value[1] || 'auto' in value[1] || 'option' in value[1]); return Array.isArray(value) && typeof value[1] === 'object' && ('scope' in value[1] || 'auto' in value[1] || 'option' in value[1] || 'timeout' in value[1]);
} }
export function isFixtureOption(value: any): value is FixtureTuple { export function isFixtureOption(value: any): value is FixtureTuple {
@ -131,12 +147,13 @@ export class FixturePool {
for (const entry of Object.entries(fixtures)) { for (const entry of Object.entries(fixtures)) {
const name = entry[0]; const name = entry[0];
let value = entry[1]; let value = entry[1];
let options: Required<FixtureOptions> | undefined; let options: { auto: boolean, scope: FixtureScope, option: boolean, timeout: number | undefined } | undefined;
if (isFixtureTuple(value)) { if (isFixtureTuple(value)) {
options = { options = {
auto: !!value[1].auto, auto: !!value[1].auto,
scope: value[1].scope || 'test', scope: value[1].scope || 'test',
option: !!value[1].option, option: !!value[1].option,
timeout: value[1].timeout,
}; };
value = value[0]; value = value[0];
} }
@ -149,9 +166,9 @@ export class FixturePool {
if (previous.auto !== options.auto) if (previous.auto !== options.auto)
throw errorWithLocations(`Fixture "${name}" has already been registered as a { auto: '${previous.scope}' } fixture.`, { location, name }, previous); throw errorWithLocations(`Fixture "${name}" has already been registered as a { auto: '${previous.scope}' } fixture.`, { location, name }, previous);
} else if (previous) { } else if (previous) {
options = { auto: previous.auto, scope: previous.scope, option: previous.option }; options = { auto: previous.auto, scope: previous.scope, option: previous.option, timeout: previous.timeout };
} else if (!options) { } else if (!options) {
options = { auto: false, scope: 'test', option: false }; options = { auto: false, scope: 'test', option: false, timeout: undefined };
} }
if (options.scope !== 'test' && options.scope !== 'worker') if (options.scope !== 'test' && options.scope !== 'worker')
@ -160,7 +177,7 @@ export class FixturePool {
throw errorWithLocations(`Cannot use({ ${name} }) in a describe group, because it forces a new worker.\nMake it top-level in the test file or put in the configuration file.`, { location, name }); throw errorWithLocations(`Cannot use({ ${name} }) in a describe group, because it forces a new worker.\nMake it top-level in the test file or put in the configuration file.`, { location, name });
const deps = fixtureParameterNames(fn, location); const deps = fixtureParameterNames(fn, location);
const registration: FixtureRegistration = { id: '', name, location, scope: options.scope, fn, auto: options.auto, option: options.option, deps, super: previous }; const registration: FixtureRegistration = { id: '', name, location, scope: options.scope, fn, auto: options.auto, option: options.option, timeout: options.timeout, deps, super: previous };
registrationId(registration); registrationId(registration);
this.registrations.set(name, registration); this.registrations.set(name, registration);
} }
@ -242,14 +259,14 @@ export class FixtureRunner {
this.pool = pool; this.pool = pool;
} }
async teardownScope(scope: FixtureScope) { async teardownScope(scope: FixtureScope, timeoutManager: TimeoutManager) {
let error: Error | undefined; let error: Error | undefined;
// Teardown fixtures in the reverse order. // Teardown fixtures in the reverse order.
const fixtures = Array.from(this.instanceForId.values()).reverse(); const fixtures = Array.from(this.instanceForId.values()).reverse();
for (const fixture of fixtures) { for (const fixture of fixtures) {
if (fixture.registration.scope === scope) { if (fixture.registration.scope === scope) {
try { try {
await fixture.teardown(); await fixture.teardown(timeoutManager);
} catch (e) { } catch (e) {
if (error === undefined) if (error === undefined)
error = e; error = e;
@ -262,7 +279,7 @@ export class FixtureRunner {
throw error; throw error;
} }
async resolveParametersForFunction(fn: Function, testInfo: TestInfo): Promise<object> { async resolveParametersForFunction(fn: Function, testInfo: TestInfoImpl): Promise<object> {
// Install all automatic fixtures. // Install all automatic fixtures.
for (const registration of this.pool!.registrations.values()) { for (const registration of this.pool!.registrations.values()) {
const shouldSkip = !testInfo && registration.scope === 'test'; const shouldSkip = !testInfo && registration.scope === 'test';
@ -281,12 +298,12 @@ export class FixtureRunner {
return params; return params;
} }
async resolveParametersAndRunFunction(fn: Function, testInfo: TestInfo) { async resolveParametersAndRunFunction(fn: Function, testInfo: TestInfoImpl) {
const params = await this.resolveParametersForFunction(fn, testInfo); const params = await this.resolveParametersForFunction(fn, testInfo);
return fn(params, testInfo); return fn(params, testInfo);
} }
async setupFixtureForRegistration(registration: FixtureRegistration, testInfo: TestInfo): Promise<Fixture> { async setupFixtureForRegistration(registration: FixtureRegistration, testInfo: TestInfoImpl): Promise<Fixture> {
if (registration.scope === 'test') if (registration.scope === 'test')
this.testScopeClean = false; this.testScopeClean = false;

View file

@ -510,9 +510,9 @@ function formatStackFrame(frame: StackFrame) {
} }
function hookType(testInfo: TestInfo): 'beforeAll' | 'afterAll' | undefined { function hookType(testInfo: TestInfo): 'beforeAll' | 'afterAll' | undefined {
if ((testInfo as any)._currentRunnable?.type === 'beforeAll') if ((testInfo as any)._timeoutManager._runnable?.type === 'beforeAll')
return 'beforeAll'; return 'beforeAll';
if ((testInfo as any)._currentRunnable?.type === 'afterAll') if ((testInfo as any)._timeoutManager._runnable?.type === 'afterAll')
return 'afterAll'; return 'afterAll';
} }

View file

@ -14,38 +14,27 @@
* limitations under the License. * limitations under the License.
*/ */
import colors from 'colors/safe';
import fs from 'fs'; import fs from 'fs';
import * as mime from 'mime'; import * as mime from 'mime';
import path from 'path'; import path from 'path';
import { TimeoutRunner, TimeoutRunnerError } from 'playwright-core/lib/utils/async';
import { calculateSha1 } from 'playwright-core/lib/utils/utils'; import { calculateSha1 } from 'playwright-core/lib/utils/utils';
import type { FullConfig, FullProject, TestError, TestInfo, TestStatus } from '../types/test'; import type { FullConfig, FullProject, TestError, TestInfo, TestStatus } from '../types/test';
import { WorkerInitParams } from './ipc'; import { WorkerInitParams } from './ipc';
import { Loader } from './loader'; import { Loader } from './loader';
import { ProjectImpl } from './project'; import { ProjectImpl } from './project';
import { TestCase } from './test'; import { TestCase } from './test';
import { Annotation, TestStepInternal, Location } from './types'; import { TimeoutManager } from './timeoutManager';
import { Annotation, TestStepInternal } from './types';
import { addSuffixToFilePath, getContainedPath, monotonicTime, sanitizeForFilePath, serializeError, trimLongString } from './util'; import { addSuffixToFilePath, getContainedPath, monotonicTime, sanitizeForFilePath, serializeError, trimLongString } from './util';
type RunnableDescription = {
type: 'test' | 'beforeAll' | 'afterAll' | 'beforeEach' | 'afterEach' | 'slow' | 'skip' | 'fail' | 'fixme' | 'teardown';
location?: Location;
// When runnable has a separate timeout, it does not count into the "shared time pool" for the test.
timeout?: number;
};
export class TestInfoImpl implements TestInfo { export class TestInfoImpl implements TestInfo {
private _projectImpl: ProjectImpl; private _projectImpl: ProjectImpl;
private _addStepImpl: (data: Omit<TestStepInternal, 'complete'>) => TestStepInternal; private _addStepImpl: (data: Omit<TestStepInternal, 'complete'>) => TestStepInternal;
readonly _test: TestCase; readonly _test: TestCase;
readonly _timeoutRunner: TimeoutRunner; readonly _timeoutManager: TimeoutManager;
readonly _startTime: number; readonly _startTime: number;
readonly _startWallTime: number; readonly _startWallTime: number;
private _hasHardError: boolean = false; private _hasHardError: boolean = false;
private _currentRunnable: RunnableDescription = { type: 'test' };
// Holds elapsed time of the "time pool" shared between fixtures, each hooks and test itself.
private _elapsedTestTime = 0;
readonly _screenshotsDir: string; readonly _screenshotsDir: string;
// ------------ TestInfo fields ------------ // ------------ TestInfo fields ------------
@ -68,7 +57,6 @@ export class TestInfoImpl implements TestInfo {
status: TestStatus = 'passed'; status: TestStatus = 'passed';
readonly stdout: TestInfo['stdout'] = []; readonly stdout: TestInfo['stdout'] = [];
readonly stderr: TestInfo['stderr'] = []; readonly stderr: TestInfo['stderr'] = [];
timeout: number;
snapshotSuffix: string = ''; snapshotSuffix: string = '';
readonly outputDir: string; readonly outputDir: string;
readonly snapshotDir: string; readonly snapshotDir: string;
@ -87,6 +75,14 @@ export class TestInfoImpl implements TestInfo {
this.errors[0] = e; this.errors[0] = e;
} }
get timeout(): number {
return this._timeoutManager.defaultTimeout();
}
set timeout(timeout: number) {
// Ignored.
}
constructor( constructor(
loader: Loader, loader: Loader,
workerParams: WorkerInitParams, workerParams: WorkerInitParams,
@ -113,9 +109,8 @@ export class TestInfoImpl implements TestInfo {
this.column = test.location.column; this.column = test.location.column;
this.fn = test.fn; this.fn = test.fn;
this.expectedStatus = test.expectedStatus; this.expectedStatus = test.expectedStatus;
this.timeout = this.project.timeout;
this._timeoutRunner = new TimeoutRunner(this.timeout); this._timeoutManager = new TimeoutManager(this.project.timeout);
this.outputDir = (() => { this.outputDir = (() => {
const sameName = loader.projects().filter(project => project.config.name === this.project.name); const sameName = loader.projects().filter(project => project.config.name === this.project.name);
@ -166,7 +161,7 @@ export class TestInfoImpl implements TestInfo {
const description = modifierArgs[1]; const description = modifierArgs[1];
this.annotations.push({ type, description }); this.annotations.push({ type, description });
if (type === 'slow') { if (type === 'slow') {
this.setTimeout(this.timeout * 3); this._timeoutManager.slow();
} else if (type === 'skip' || type === 'fixme') { } else if (type === 'skip' || type === 'fixme') {
this.expectedStatus = 'skipped'; this.expectedStatus = 'skipped';
throw new SkipError('Test is skipped: ' + (description || '')); throw new SkipError('Test is skipped: ' + (description || ''));
@ -176,35 +171,12 @@ export class TestInfoImpl implements TestInfo {
} }
} }
_setCurrentRunnable(runnable: RunnableDescription) {
if (this._currentRunnable.timeout === undefined)
this._elapsedTestTime = this._timeoutRunner.elapsed();
this._currentRunnable = runnable;
if (runnable.timeout === undefined)
this._timeoutRunner.updateTimeout(this.timeout, this._elapsedTestTime);
else
this._timeoutRunner.updateTimeout(runnable.timeout, 0);
}
async _runWithTimeout(cb: () => Promise<any>): Promise<void> { async _runWithTimeout(cb: () => Promise<any>): Promise<void> {
try { const timeoutError = await this._timeoutManager.runWithTimeout(cb);
await this._timeoutRunner.run(cb); // Do not overwrite existing failure upon hook/teardown timeout.
} catch (error) { if (timeoutError && this.status === 'passed') {
if (!(error instanceof TimeoutRunnerError)) this.status = 'timedOut';
throw error; this.errors.push(timeoutError);
// Do not overwrite existing failure upon hook/teardown timeout.
if (this.status === 'passed') {
this.status = 'timedOut';
const title = titleForRunnable(this._currentRunnable);
const suffix = title ? ` in ${title}` : '';
const message = colors.red(`Timeout of ${this._currentRunnable.timeout ?? this.timeout}ms exceeded${suffix}.`);
const location = this._currentRunnable.location;
this.errors.push({
message,
// Include location for hooks and modifiers to distinguish between them.
stack: location ? message + `\n at ${location.file}:${location.line}:${location.column}` : undefined,
});
}
} }
this.duration = monotonicTime() - this._startTime; this.duration = monotonicTime() - this._startTime;
} }
@ -308,38 +280,10 @@ export class TestInfoImpl implements TestInfo {
} }
setTimeout(timeout: number) { setTimeout(timeout: number) {
if (this._currentRunnable.timeout !== undefined) { this._timeoutManager.setTimeout(timeout);
if (!this._currentRunnable.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
this._currentRunnable.timeout = timeout;
this._timeoutRunner.updateTimeout(timeout);
} else {
if (!this.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
this.timeout = timeout;
this._timeoutRunner.updateTimeout(timeout);
}
} }
} }
class SkipError extends Error { class SkipError extends Error {
} }
function titleForRunnable(runnable: RunnableDescription): string {
switch (runnable.type) {
case 'test':
return '';
case 'beforeAll':
case 'beforeEach':
case 'afterAll':
case 'afterEach':
return runnable.type + ' hook';
case 'teardown':
return 'fixtures teardown';
case 'skip':
case 'slow':
case 'fixme':
case 'fail':
return runnable.type + ' modifier';
}
}

View file

@ -0,0 +1,134 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import { TimeoutRunner, TimeoutRunnerError } from 'playwright-core/lib/utils/async';
import type { TestError } from '../types/test';
import { Location } from './types';
export type TimeSlot = {
timeout: number;
elapsed: number;
};
type RunnableDescription = {
type: 'test' | 'beforeAll' | 'afterAll' | 'beforeEach' | 'afterEach' | 'slow' | 'skip' | 'fail' | 'fixme' | 'teardown';
location?: Location;
slot?: TimeSlot; // Falls back to test slot.
};
export type FixtureDescription = {
fixture: string;
location?: Location;
slot?: TimeSlot; // Falls back to current runnable slot.
};
export class TimeoutManager {
private _defaultSlot: TimeSlot;
private _runnable: RunnableDescription;
private _fixture: FixtureDescription | undefined;
private _timeoutRunner: TimeoutRunner;
constructor(timeout: number) {
this._defaultSlot = { timeout, elapsed: 0 };
this._runnable = { type: 'test', slot: this._defaultSlot };
this._timeoutRunner = new TimeoutRunner(timeout);
}
interrupt() {
this._timeoutRunner.interrupt();
}
setCurrentRunnable(runnable: RunnableDescription) {
this._updateRunnables(runnable, undefined);
}
setCurrentFixture(fixture: FixtureDescription | undefined) {
this._updateRunnables(this._runnable, fixture);
}
defaultTimeout() {
return this._defaultSlot.timeout;
}
slow() {
const slot = this._currentSlot();
slot.timeout = slot.timeout * 3;
this._timeoutRunner.updateTimeout(slot.timeout);
}
async runWithTimeout(cb: () => Promise<any>): Promise<TestError | undefined> {
try {
await this._timeoutRunner.run(cb);
} catch (error) {
if (!(error instanceof TimeoutRunnerError))
throw error;
return this._createTimeoutError();
}
}
setTimeout(timeout: number) {
const slot = this._currentSlot();
if (!slot.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
slot.timeout = timeout;
this._timeoutRunner.updateTimeout(timeout);
}
private _currentSlot() {
return this._fixture?.slot || this._runnable.slot || this._defaultSlot;
}
private _updateRunnables(runnable: RunnableDescription, fixture: FixtureDescription | undefined) {
let slot = this._currentSlot();
slot.elapsed = this._timeoutRunner.elapsed();
this._runnable = runnable;
this._fixture = fixture;
slot = this._currentSlot();
this._timeoutRunner.updateTimeout(slot.timeout, slot.elapsed);
}
private _createTimeoutError(): TestError {
let suffix = '';
switch (this._runnable.type) {
case 'test':
suffix = ''; break;
case 'beforeAll':
case 'beforeEach':
case 'afterAll':
case 'afterEach':
suffix = ` in ${this._runnable.type} hook`; break;
case 'teardown':
suffix = ` in fixtures teardown`; break;
case 'skip':
case 'slow':
case 'fixme':
case 'fail':
suffix = ` in ${this._runnable.type} modifier`; break;
}
if (this._fixture && this._fixture.slot)
suffix = ` in fixture "${this._fixture.fixture}"`;
const message = colors.red(`Timeout of ${this._currentSlot().timeout}ms exceeded${suffix}.`);
const location = (this._fixture || this._runnable).location;
return {
message,
// Include location for hooks, modifiers and fixtures to distinguish between them.
stack: location ? message + `\n at ${location.file}:${location.line}:${location.column}` : undefined,
};
}
}

View file

@ -16,7 +16,6 @@
import rimraf from 'rimraf'; import rimraf from 'rimraf';
import util from 'util'; import util from 'util';
import colors from 'colors/safe';
import { EventEmitter } from 'events'; import { EventEmitter } from 'events';
import { serializeError } from './util'; import { serializeError } from './util';
import { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload, TeardownErrorsPayload } from './ipc'; import { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload, TeardownErrorsPayload } from './ipc';
@ -26,8 +25,9 @@ import { Suite, TestCase } from './test';
import { Annotation, TestError, TestStepInternal } from './types'; import { Annotation, TestError, TestStepInternal } from './types';
import { ProjectImpl } from './project'; import { ProjectImpl } from './project';
import { FixtureRunner } from './fixtures'; import { FixtureRunner } from './fixtures';
import { ManualPromise, raceAgainstTimeout } from 'playwright-core/lib/utils/async'; import { ManualPromise } from 'playwright-core/lib/utils/async';
import { TestInfoImpl } from './testInfo'; import { TestInfoImpl } from './testInfo';
import { TimeoutManager, TimeSlot } from './timeoutManager';
const removeFolderAsync = util.promisify(rimraf); const removeFolderAsync = util.promisify(rimraf);
@ -70,7 +70,7 @@ export class WorkerRunner extends EventEmitter {
this._isStopped = true; this._isStopped = true;
// Interrupt current action. // Interrupt current action.
this._currentTest?._timeoutRunner.interrupt(); this._currentTest?._timeoutManager.interrupt();
// TODO: mark test as 'interrupted' instead. // TODO: mark test as 'interrupted' instead.
if (this._currentTest && this._currentTest.status === 'passed') if (this._currentTest && this._currentTest.status === 'passed')
@ -91,12 +91,14 @@ export class WorkerRunner extends EventEmitter {
private async _teardownScopes() { private async _teardownScopes() {
// TODO: separate timeout for teardown? // TODO: separate timeout for teardown?
const result = await raceAgainstTimeout(async () => { const timeoutManager = new TimeoutManager(this._project.config.timeout);
await this._fixtureRunner.teardownScope('test'); timeoutManager.setCurrentRunnable({ type: 'teardown' });
await this._fixtureRunner.teardownScope('worker'); const timeoutError = await timeoutManager.runWithTimeout(async () => {
}, this._project.config.timeout); await this._fixtureRunner.teardownScope('test', timeoutManager);
if (result.timedOut) await this._fixtureRunner.teardownScope('worker', timeoutManager);
this._fatalErrors.push({ message: colors.red(`Timeout of ${this._project.config.timeout}ms exceeded while shutting down environment`) }); });
if (timeoutError)
this._fatalErrors.push(timeoutError);
} }
unhandledError(error: Error | any) { unhandledError(error: Error | any) {
@ -226,7 +228,7 @@ export class WorkerRunner extends EventEmitter {
testInfo.expectedStatus = 'failed'; testInfo.expectedStatus = 'failed';
break; break;
case 'slow': case 'slow':
testInfo.setTimeout(testInfo.timeout * 3); testInfo.slow();
break; break;
} }
}; };
@ -242,7 +244,7 @@ export class WorkerRunner extends EventEmitter {
// Inherit test.setTimeout() from parent suites, deepest has the priority. // Inherit test.setTimeout() from parent suites, deepest has the priority.
for (const suite of reversedSuites) { for (const suite of reversedSuites) {
if (suite._timeout !== undefined) { if (suite._timeout !== undefined) {
testInfo.setTimeout(suite._timeout); testInfo._timeoutManager.setTimeout(suite._timeout);
break; break;
} }
} }
@ -295,7 +297,9 @@ export class WorkerRunner extends EventEmitter {
const extraAnnotations: Annotation[] = []; const extraAnnotations: Annotation[] = [];
this._extraSuiteAnnotations.set(suite, extraAnnotations); this._extraSuiteAnnotations.set(suite, extraAnnotations);
didFailBeforeAllForSuite = suite; // Assume failure, unless reset below. didFailBeforeAllForSuite = suite; // Assume failure, unless reset below.
await this._runModifiersForSuite(suite, testInfo, 'worker', extraAnnotations); // Separate timeout for each "beforeAll" modifier.
const timeSlot = { timeout: this._project.config.timeout, elapsed: 0 };
await this._runModifiersForSuite(suite, testInfo, 'worker', timeSlot, extraAnnotations);
} }
// Run "beforeAll" hooks, unless already run during previous tests. // Run "beforeAll" hooks, unless already run during previous tests.
@ -309,14 +313,14 @@ export class WorkerRunner extends EventEmitter {
// Run "beforeEach" modifiers. // Run "beforeEach" modifiers.
for (const suite of suites) for (const suite of suites)
await this._runModifiersForSuite(suite, testInfo, 'test'); await this._runModifiersForSuite(suite, testInfo, 'test', undefined);
// Run "beforeEach" hooks. Once started with "beforeEach", we must run all "afterEach" hooks as well. // Run "beforeEach" hooks. Once started with "beforeEach", we must run all "afterEach" hooks as well.
shouldRunAfterEachHooks = true; shouldRunAfterEachHooks = true;
await this._runEachHooksForSuites(suites, 'beforeEach', testInfo); await this._runEachHooksForSuites(suites, 'beforeEach', testInfo, undefined);
// Setup fixtures required by the test. // Setup fixtures required by the test.
testInfo._setCurrentRunnable({ type: 'test' }); testInfo._timeoutManager.setCurrentRunnable({ type: 'test' });
const params = await this._fixtureRunner.resolveParametersForFunction(test.fn, testInfo); const params = await this._fixtureRunner.resolveParametersForFunction(test.fn, testInfo);
beforeHooksStep.complete(); // Report fixture hooks step as completed. beforeHooksStep.complete(); // Report fixture hooks step as completed.
@ -343,9 +347,10 @@ export class WorkerRunner extends EventEmitter {
}); });
let firstAfterHooksError: TestError | undefined; let firstAfterHooksError: TestError | undefined;
let afterHooksSlot: TimeSlot | undefined;
if (testInfo.status === 'timedOut') { if (testInfo.status === 'timedOut') {
// A timed-out test gets a full additional timeout to run after hooks. // A timed-out test gets a full additional timeout to run after hooks.
testInfo._timeoutRunner.updateTimeout(testInfo.timeout, 0); afterHooksSlot = { timeout: this._project.config.timeout, elapsed: 0 };
} }
await testInfo._runWithTimeout(async () => { await testInfo._runWithTimeout(async () => {
// Note: do not wrap all teardown steps together, because failure in any of them // Note: do not wrap all teardown steps together, because failure in any of them
@ -353,7 +358,7 @@ export class WorkerRunner extends EventEmitter {
// Run "afterEach" hooks, unless we failed at beforeAll stage. // Run "afterEach" hooks, unless we failed at beforeAll stage.
if (shouldRunAfterEachHooks) { if (shouldRunAfterEachHooks) {
const afterEachError = await testInfo._runFn(() => this._runEachHooksForSuites(reversedSuites, 'afterEach', testInfo)); const afterEachError = await testInfo._runFn(() => this._runEachHooksForSuites(reversedSuites, 'afterEach', testInfo, afterHooksSlot));
firstAfterHooksError = firstAfterHooksError || afterEachError; firstAfterHooksError = firstAfterHooksError || afterEachError;
} }
@ -367,8 +372,8 @@ export class WorkerRunner extends EventEmitter {
} }
// Teardown test-scoped fixtures. // Teardown test-scoped fixtures.
testInfo._setCurrentRunnable({ type: 'teardown' }); testInfo._timeoutManager.setCurrentRunnable({ type: 'teardown', slot: afterHooksSlot });
const testScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test')); const testScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test', testInfo._timeoutManager));
firstAfterHooksError = firstAfterHooksError || testScopeError; firstAfterHooksError = firstAfterHooksError || testScopeError;
}); });
@ -382,16 +387,16 @@ export class WorkerRunner extends EventEmitter {
this._didRunFullCleanup = true; this._didRunFullCleanup = true;
// Give it more time for the full cleanup. // Give it more time for the full cleanup.
testInfo._timeoutRunner.updateTimeout(this._project.config.timeout, 0);
await testInfo._runWithTimeout(async () => { await testInfo._runWithTimeout(async () => {
for (const suite of reversedSuites) { for (const suite of reversedSuites) {
const afterAllError = await this._runAfterAllHooksForSuite(suite, testInfo); const afterAllError = await this._runAfterAllHooksForSuite(suite, testInfo);
firstAfterHooksError = firstAfterHooksError || afterAllError; firstAfterHooksError = firstAfterHooksError || afterAllError;
} }
testInfo._setCurrentRunnable({ type: 'teardown', timeout: this._project.config.timeout }); const teardownSlot = { timeout: this._project.config.timeout, elapsed: 0 };
const testScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test')); testInfo._timeoutManager.setCurrentRunnable({ type: 'teardown', slot: teardownSlot });
const testScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test', testInfo._timeoutManager));
firstAfterHooksError = firstAfterHooksError || testScopeError; firstAfterHooksError = firstAfterHooksError || testScopeError;
const workerScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('worker')); const workerScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('worker', testInfo._timeoutManager));
firstAfterHooksError = firstAfterHooksError || workerScopeError; firstAfterHooksError = firstAfterHooksError || workerScopeError;
}); });
} }
@ -407,12 +412,12 @@ export class WorkerRunner extends EventEmitter {
await removeFolderAsync(testInfo.outputDir).catch(e => {}); await removeFolderAsync(testInfo.outputDir).catch(e => {});
} }
private async _runModifiersForSuite(suite: Suite, testInfo: TestInfoImpl, scope: 'worker' | 'test', extraAnnotations?: Annotation[]) { private async _runModifiersForSuite(suite: Suite, testInfo: TestInfoImpl, scope: 'worker' | 'test', timeSlot: TimeSlot | undefined, extraAnnotations?: Annotation[]) {
for (const modifier of suite._modifiers) { for (const modifier of suite._modifiers) {
const actualScope = this._fixtureRunner.dependsOnWorkerFixturesOnly(modifier.fn, modifier.location) ? 'worker' : 'test'; const actualScope = this._fixtureRunner.dependsOnWorkerFixturesOnly(modifier.fn, modifier.location) ? 'worker' : 'test';
if (actualScope !== scope) if (actualScope !== scope)
continue; continue;
testInfo._setCurrentRunnable({ type: modifier.type, location: modifier.location, timeout: scope === 'worker' ? this._project.config.timeout : undefined }); testInfo._timeoutManager.setCurrentRunnable({ type: modifier.type, location: modifier.location, slot: timeSlot });
const result = await this._fixtureRunner.resolveParametersAndRunFunction(modifier.fn, testInfo); const result = await this._fixtureRunner.resolveParametersAndRunFunction(modifier.fn, testInfo);
if (result && extraAnnotations) if (result && extraAnnotations)
extraAnnotations.push({ type: modifier.type, description: modifier.description }); extraAnnotations.push({ type: modifier.type, description: modifier.description });
@ -429,7 +434,9 @@ export class WorkerRunner extends EventEmitter {
if (hook.type !== 'beforeAll') if (hook.type !== 'beforeAll')
continue; continue;
try { try {
testInfo._setCurrentRunnable({ type: 'beforeAll', location: hook.location, timeout: this._project.config.timeout }); // Separate time slot for each "beforeAll" hook.
const timeSlot = { timeout: this._project.config.timeout, elapsed: 0 };
testInfo._timeoutManager.setCurrentRunnable({ type: 'beforeAll', location: hook.location, slot: timeSlot });
await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo); await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo);
} catch (e) { } catch (e) {
// Always run all the hooks, and capture the first error. // Always run all the hooks, and capture the first error.
@ -449,7 +456,9 @@ export class WorkerRunner extends EventEmitter {
if (hook.type !== 'afterAll') if (hook.type !== 'afterAll')
continue; continue;
const afterAllError = await testInfo._runFn(async () => { const afterAllError = await testInfo._runFn(async () => {
testInfo._setCurrentRunnable({ type: 'afterAll', location: hook.location, timeout: this._project.config.timeout }); // Separate time slot for each "afterAll" hook.
const timeSlot = { timeout: this._project.config.timeout, elapsed: 0 };
testInfo._timeoutManager.setCurrentRunnable({ type: 'afterAll', location: hook.location, slot: timeSlot });
await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo); await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo);
}); });
firstError = firstError || afterAllError; firstError = firstError || afterAllError;
@ -457,12 +466,12 @@ export class WorkerRunner extends EventEmitter {
return firstError; return firstError;
} }
private async _runEachHooksForSuites(suites: Suite[], type: 'beforeEach' | 'afterEach', testInfo: TestInfoImpl) { private async _runEachHooksForSuites(suites: Suite[], type: 'beforeEach' | 'afterEach', testInfo: TestInfoImpl, timeSlot: TimeSlot | undefined) {
const hooks = suites.map(suite => suite._hooks.filter(hook => hook.type === type)).flat(); const hooks = suites.map(suite => suite._hooks.filter(hook => hook.type === type)).flat();
let error: Error | undefined; let error: Error | undefined;
for (const hook of hooks) { for (const hook of hooks) {
try { try {
testInfo._setCurrentRunnable({ type, location: hook.location }); testInfo._timeoutManager.setCurrentRunnable({ type, location: hook.location, slot: timeSlot });
await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo); await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo);
} catch (e) { } catch (e) {
// Always run all the hooks, and capture the first error. // Always run all the hooks, and capture the first error.

View file

@ -2533,7 +2533,7 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
*/ */
slow(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void; slow(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void;
/** /**
* Changes the timeout for the test. Learn more about [various timeouts](https://playwright.dev/docs/test-timeouts). * Changes the timeout for the test. Zero means no timeout. Learn more about [various timeouts](https://playwright.dev/docs/test-timeouts).
* *
* ```ts * ```ts
* import { test, expect } from '@playwright/test'; * import { test, expect } from '@playwright/test';
@ -2797,13 +2797,13 @@ export type WorkerFixture<R, Args extends KeyValue> = (args: Args, use: (r: R) =
type TestFixtureValue<R, Args> = Exclude<R, Function> | TestFixture<R, Args>; type TestFixtureValue<R, Args> = Exclude<R, Function> | TestFixture<R, Args>;
type WorkerFixtureValue<R, Args> = Exclude<R, Function> | WorkerFixture<R, Args>; type WorkerFixtureValue<R, Args> = Exclude<R, Function> | WorkerFixture<R, Args>;
export type Fixtures<T extends KeyValue = {}, W extends KeyValue = {}, PT extends KeyValue = {}, PW extends KeyValue = {}> = { export type Fixtures<T extends KeyValue = {}, W extends KeyValue = {}, PT extends KeyValue = {}, PW extends KeyValue = {}> = {
[K in keyof PW]?: WorkerFixtureValue<PW[K], W & PW> | [WorkerFixtureValue<PW[K], W & PW>, { scope: 'worker' }]; [K in keyof PW]?: WorkerFixtureValue<PW[K], W & PW> | [WorkerFixtureValue<PW[K], W & PW>, { scope: 'worker', timeout?: number | undefined }];
} & { } & {
[K in keyof PT]?: TestFixtureValue<PT[K], T & W & PT & PW> | [TestFixtureValue<PT[K], T & W & PT & PW>, { scope: 'test' }]; [K in keyof PT]?: TestFixtureValue<PT[K], T & W & PT & PW> | [TestFixtureValue<PT[K], T & W & PT & PW>, { scope: 'test', timeout?: number | undefined }];
} & { } & {
[K in keyof W]?: [WorkerFixtureValue<W[K], W & PW>, { scope: 'worker', auto?: boolean, option?: boolean }]; [K in keyof W]?: [WorkerFixtureValue<W[K], W & PW>, { scope: 'worker', auto?: boolean, option?: boolean, timeout?: number | undefined }];
} & { } & {
[K in keyof T]?: TestFixtureValue<T[K], T & W & PT & PW> | [TestFixtureValue<T[K], T & W & PT & PW>, { scope?: 'test', auto?: boolean, option?: boolean }]; [K in keyof T]?: TestFixtureValue<T[K], T & W & PT & PW> | [TestFixtureValue<T[K], T & W & PT & PW>, { scope?: 'test', auto?: boolean, option?: boolean, timeout?: number | undefined }];
}; };
type BrowserName = 'chromium' | 'firefox' | 'webkit'; type BrowserName = 'chromium' | 'firefox' | 'webkit';

View file

@ -456,7 +456,7 @@ test('should not report fixture teardown error twice', async ({ runInlineTest })
expect(countTimes(stripAnsi(result.output), 'Oh my error')).toBe(2); expect(countTimes(stripAnsi(result.output), 'Oh my error')).toBe(2);
}); });
test('should not report fixture teardown timeout twice', async ({ runInlineTest }) => { test.fixme('should not report fixture teardown timeout twice', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.spec.ts': ` 'a.spec.ts': `
const test = pwt.test.extend({ const test = pwt.test.extend({
@ -471,8 +471,8 @@ test('should not report fixture teardown timeout twice', async ({ runInlineTest
}, { reporter: 'list', timeout: 1000 }); }, { reporter: 'list', timeout: 1000 });
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1); expect(result.failed).toBe(1);
expect(result.output).toContain('while shutting down environment'); expect(result.output).toContain('in fixtures teardown');
expect(countTimes(result.output, 'while shutting down environment')).toBe(1); expect(countTimes(result.output, 'in fixtures teardown')).toBe(1);
}); });
test('should handle fixture teardown error after test timeout and continue', async ({ runInlineTest }) => { test('should handle fixture teardown error after test timeout and continue', async ({ runInlineTest }) => {

View file

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
import { test, expect } from './playwright-test-fixtures'; import { test, expect, stripAnsi } from './playwright-test-fixtures';
test('should run fixture teardown on timeout', async ({ runInlineTest }) => { test('should run fixture teardown on timeout', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
@ -144,8 +144,14 @@ test('should respect test.slow', async ({ runInlineTest }) => {
test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) => { test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.spec.ts': ` 'a.spec.ts': `
const { test } = pwt; const test = pwt.test.extend({
test('my test', async ({}) => { fixture: async ({}, use) => {
test.setTimeout(100);
await new Promise(f => setTimeout(f, 200));
await use('hey');
},
});
test('my test', async ({ fixture }) => {
test.setTimeout(1000); test.setTimeout(1000);
await new Promise(f => setTimeout(f, 2000)); await new Promise(f => setTimeout(f, 2000));
}); });
@ -154,3 +160,149 @@ test('should ignore test.setTimeout when debugging', async ({ runInlineTest }) =
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1); expect(result.passed).toBe(1);
}); });
test('should respect fixture timeout', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
const test = pwt.test.extend({
fixture: [async ({}, use) => {
await new Promise(f => setTimeout(f, 300));
await use('hey');
await new Promise(f => setTimeout(f, 300));
}, { timeout: 1000 }],
noTimeout: [async ({}, use) => {
await new Promise(f => setTimeout(f, 300));
await use('hey');
await new Promise(f => setTimeout(f, 300));
}, { timeout: 0 }],
slowSetup: [async ({}, use) => {
await new Promise(f => setTimeout(f, 2000));
await use('hey');
}, { timeout: 500 }],
slowTeardown: [async ({}, use) => {
await use('hey');
await new Promise(f => setTimeout(f, 2000));
}, { timeout: 400 }],
});
test('test ok', async ({ fixture, noTimeout }) => {
await new Promise(f => setTimeout(f, 1000));
});
test('test setup', async ({ slowSetup }) => {
});
test('test teardown', async ({ slowTeardown }) => {
});
`
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(1);
expect(result.failed).toBe(2);
expect(result.output).toContain('Timeout of 500ms exceeded in fixture "slowSetup"');
expect(result.output).toContain('Timeout of 400ms exceeded in fixture "slowTeardown"');
expect(stripAnsi(result.output)).toContain('> 5 | const test = pwt.test.extend({');
});
test('should respect test.setTimeout in the worker fixture', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
const test = pwt.test.extend({
fixture: [async ({}, use) => {
await new Promise(f => setTimeout(f, 300));
await use('hey');
await new Promise(f => setTimeout(f, 300));
}, { scope: 'worker', timeout: 1000 }],
noTimeout: [async ({}, use) => {
await new Promise(f => setTimeout(f, 300));
await use('hey');
await new Promise(f => setTimeout(f, 300));
}, { scope: 'worker', timeout: 0 }],
slowSetup: [async ({}, use) => {
await new Promise(f => setTimeout(f, 2000));
await use('hey');
}, { scope: 'worker', timeout: 500 }],
slowTeardown: [async ({}, use) => {
await use('hey');
await new Promise(f => setTimeout(f, 2000));
}, { scope: 'worker', timeout: 400 }],
});
test('test ok', async ({ fixture, noTimeout }) => {
await new Promise(f => setTimeout(f, 1000));
});
test('test setup', async ({ slowSetup }) => {
});
test('test teardown', async ({ slowTeardown }) => {
});
`
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(2);
expect(result.failed).toBe(1);
expect(result.output).toContain('Timeout of 500ms exceeded in fixture "slowSetup"');
expect(result.output).toContain('Timeout of 400ms exceeded in fixture "slowTeardown"');
});
test('fixture time in beforeAll hook should not affect test', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
const test = pwt.test.extend({
fixture: async ({}, use) => {
await new Promise(f => setTimeout(f, 500));
await use('hey');
},
});
test.beforeAll(async ({ fixture }) => {
// Nothing to see here.
});
test('test ok', async ({}) => {
test.setTimeout(1000);
await new Promise(f => setTimeout(f, 800));
});
`
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
});
test('fixture timeout in beforeAll hook should not affect test', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
const test = pwt.test.extend({
fixture: [async ({}, use) => {
await new Promise(f => setTimeout(f, 500));
await use('hey');
}, { timeout: 800 }],
});
test.beforeAll(async ({ fixture }) => {
// Nothing to see here.
});
test('test ok', async ({}) => {
test.setTimeout(1000);
await new Promise(f => setTimeout(f, 800));
});
`
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
});
test('fixture time in beforeEach hook should affect test', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
const test = pwt.test.extend({
fixture: async ({}, use) => {
await new Promise(f => setTimeout(f, 500));
await use('hey');
},
});
test.beforeEach(async ({ fixture }) => {
// Nothing to see here.
});
test('test ok', async ({}) => {
test.setTimeout(1000);
await new Promise(f => setTimeout(f, 800));
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Timeout of 1000ms exceeded');
});

View file

@ -22,7 +22,7 @@ test('should check types of fixtures', async ({ runTSC }) => {
export type MyOptions = { foo: string, bar: number }; export type MyOptions = { foo: string, bar: number };
export const test = pwt.test.extend<{ foo: string }, { bar: number }>({ export const test = pwt.test.extend<{ foo: string }, { bar: number }>({
foo: 'foo', foo: 'foo',
bar: [ 42, { scope: 'worker' } ], bar: [ 42, { scope: 'worker', timeout: 123 } ],
}); });
const good1 = test.extend<{}>({ foo: async ({ bar }, run) => run('foo') }); const good1 = test.extend<{}>({ foo: async ({ bar }, run) => run('foo') });
@ -35,7 +35,7 @@ test('should check types of fixtures', async ({ runTSC }) => {
foo: async ({ baz }, run) => run('foo') foo: async ({ baz }, run) => run('foo')
}); });
const good7 = test.extend<{ baz: boolean }>({ const good7 = test.extend<{ baz: boolean }>({
baz: [ false, { auto: true } ], baz: [ false, { auto: true, timeout: 0 } ],
}); });
const good8 = test.extend<{ foo: string }>({ const good8 = test.extend<{ foo: string }>({
foo: [ async ({}, use) => { foo: [ async ({}, use) => {
@ -82,6 +82,12 @@ test('should check types of fixtures', async ({ runTSC }) => {
// @ts-expect-error // @ts-expect-error
}, { scope: 'test' } ], }, { scope: 'test' } ],
}); });
const fail11 = test.extend<{ yay: string }>({
yay: [ async ({}, use) => {
await use('foo');
// @ts-expect-error
}, { scope: 'test', timeout: 'str' } ],
});
type AssertNotAny<S> = {notRealProperty: number} extends S ? false : true; type AssertNotAny<S> = {notRealProperty: number} extends S ? false : true;
type AssertType<T, S> = S extends T ? AssertNotAny<S> : false; type AssertType<T, S> = S extends T ? AssertNotAny<S> : false;

View file

@ -334,13 +334,13 @@ export type WorkerFixture<R, Args extends KeyValue> = (args: Args, use: (r: R) =
type TestFixtureValue<R, Args> = Exclude<R, Function> | TestFixture<R, Args>; type TestFixtureValue<R, Args> = Exclude<R, Function> | TestFixture<R, Args>;
type WorkerFixtureValue<R, Args> = Exclude<R, Function> | WorkerFixture<R, Args>; type WorkerFixtureValue<R, Args> = Exclude<R, Function> | WorkerFixture<R, Args>;
export type Fixtures<T extends KeyValue = {}, W extends KeyValue = {}, PT extends KeyValue = {}, PW extends KeyValue = {}> = { export type Fixtures<T extends KeyValue = {}, W extends KeyValue = {}, PT extends KeyValue = {}, PW extends KeyValue = {}> = {
[K in keyof PW]?: WorkerFixtureValue<PW[K], W & PW> | [WorkerFixtureValue<PW[K], W & PW>, { scope: 'worker' }]; [K in keyof PW]?: WorkerFixtureValue<PW[K], W & PW> | [WorkerFixtureValue<PW[K], W & PW>, { scope: 'worker', timeout?: number | undefined }];
} & { } & {
[K in keyof PT]?: TestFixtureValue<PT[K], T & W & PT & PW> | [TestFixtureValue<PT[K], T & W & PT & PW>, { scope: 'test' }]; [K in keyof PT]?: TestFixtureValue<PT[K], T & W & PT & PW> | [TestFixtureValue<PT[K], T & W & PT & PW>, { scope: 'test', timeout?: number | undefined }];
} & { } & {
[K in keyof W]?: [WorkerFixtureValue<W[K], W & PW>, { scope: 'worker', auto?: boolean, option?: boolean }]; [K in keyof W]?: [WorkerFixtureValue<W[K], W & PW>, { scope: 'worker', auto?: boolean, option?: boolean, timeout?: number | undefined }];
} & { } & {
[K in keyof T]?: TestFixtureValue<T[K], T & W & PT & PW> | [TestFixtureValue<T[K], T & W & PT & PW>, { scope?: 'test', auto?: boolean, option?: boolean }]; [K in keyof T]?: TestFixtureValue<T[K], T & W & PT & PW> | [TestFixtureValue<T[K], T & W & PT & PW>, { scope?: 'test', auto?: boolean, option?: boolean, timeout?: number | undefined }];
}; };
type BrowserName = 'chromium' | 'firefox' | 'webkit'; type BrowserName = 'chromium' | 'firefox' | 'webkit';