Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

runWithTimeout inside runAsStage

Browse source
This commit is contained in:
Dmitry Gozman 2024-02-22 16:20:00 -08:00
parent b9bd714f3b
commit 24a15ddf25
7 changed files with 219 additions and 181 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/playwright-core/src/utils/timeoutRunner.ts
View file

@ -45,11 +45,11 @@ export class TimeoutRunner {
timeoutPromise: new ManualPromise(), timeoutPromise: new ManualPromise(),
}; };
try { try {
this._updateTimeout(running, this._timeout);
const resultPromise = Promise.race([ const resultPromise = Promise.race([
cb(), cb(),
running.timeoutPromise running.timeoutPromise
]); ]);
this._updateTimeout(running, this._timeout);
return await resultPromise; return await resultPromise;
} finally { } finally {
this._updateTimeout(running, 0); this._updateTimeout(running, 0);

22
packages/playwright/src/worker/fixtureRunner.ts
View file

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
import { formatLocation, debugTest, filterStackFile } from '../util'; import { formatLocation, filterStackFile } from '../util';
import { ManualPromise } from 'playwright-core/lib/utils'; import { ManualPromise } from 'playwright-core/lib/utils';
import type { TestInfoImpl } from './testInfo'; import type { TestInfoImpl } from './testInfo';
import type { FixtureDescription } from './timeoutManager'; import type { FixtureDescription } from './timeoutManager';
@ -66,8 +66,10 @@ class Fixture {
} }
await testInfo._runAsStage({ await testInfo._runAsStage({
title: `fixture: ${this.registration.name}`,
canTimeout: true,
location: this._isInternalFixture ? this.registration.location : undefined, location: this._isInternalFixture ? this.registration.location : undefined,
stepInfo: this._shouldGenerateStep ? { title: `fixture: ${this.registration.name}`, category: 'fixture' } : undefined, stepCategory: this._shouldGenerateStep ? 'fixture' : undefined,
}, async () => { }, async () => {
testInfo._timeoutManager.setCurrentFixture(this._setupDescription); testInfo._timeoutManager.setCurrentFixture(this._setupDescription);
await this._setupInternal(testInfo); await this._setupInternal(testInfo);
@ -99,7 +101,6 @@ class Fixture {
let called = false; let called = false;
const useFuncStarted = new ManualPromise<void>(); const useFuncStarted = new ManualPromise<void>();
debugTest(`setup ${this.registration.name}`);
const useFunc = async (value: any) => { const useFunc = async (value: any) => {
if (called) if (called)
throw new Error(`Cannot provide fixture value for the second time`); throw new Error(`Cannot provide fixture value for the second time`);
@ -128,8 +129,10 @@ class Fixture {
async teardown(testInfo: TestInfoImpl) { async teardown(testInfo: TestInfoImpl) {
await testInfo._runAsStage({ await testInfo._runAsStage({
title: `fixture: ${this.registration.name}`,
canTimeout: true,
location: this._isInternalFixture ? this.registration.location : undefined, location: this._isInternalFixture ? this.registration.location : undefined,
stepInfo: this._shouldGenerateStep ? { title: `fixture: ${this.registration.name}`, category: 'fixture' } : undefined, stepCategory: this._shouldGenerateStep ? 'fixture' : undefined,
}, async () => { }, async () => {
testInfo._timeoutManager.setCurrentFixture(this._teardownDescription); testInfo._timeoutManager.setCurrentFixture(this._teardownDescription);
if (!this._teardownWithDepsComplete) if (!this._teardownWithDepsComplete)
@ -149,7 +152,6 @@ class Fixture {
this._usages.clear(); this._usages.clear();
} }
if (this._useFuncFinished) { if (this._useFuncFinished) {
debugTest(`teardown ${this.registration.name}`);
this._useFuncFinished.resolve(); this._useFuncFinished.resolve();
await this._selfTeardownComplete; await this._selfTeardownComplete;
} }
@ -204,12 +206,12 @@ export class FixtureRunner {
const collector = new Set<Fixture>(); const collector = new Set<Fixture>();
for (const fixture of fixtures) for (const fixture of fixtures)
fixture._collectFixturesInTeardownOrder(scope, collector); fixture._collectFixturesInTeardownOrder(scope, collector);
await testInfo._runAsStage({}, async () => { await testInfo._runAsStage({ title: `teardown ${scope} scope` }, async () => {
for (const fixture of collector) for (const fixture of collector)
await fixture.teardown(testInfo); await fixture.teardown(testInfo);
if (scope === 'test')
this.testScopeClean = true;
}); });
if (scope === 'test')
this.testScopeClean = true;
} }
async resolveParametersForFunction(fn: Function, testInfo: TestInfoImpl, autoFixtures: 'worker' | 'test' | 'all-hooks-only'): Promise<object | null> { async resolveParametersForFunction(fn: Function, testInfo: TestInfoImpl, autoFixtures: 'worker' | 'test' | 'all-hooks-only'): Promise<object | null> {
@ -236,7 +238,7 @@ export class FixtureRunner {
this._collectFixturesInSetupOrder(this.pool!.resolve(name)!, collector); this._collectFixturesInSetupOrder(this.pool!.resolve(name)!, collector);
// Setup fixtures. // Setup fixtures.
await testInfo._runAsStage({ stopOnChildError: true }, async () => { await testInfo._runAsStage({ title: 'setup fixtures', stopOnChildError: true }, async () => {
for (const registration of collector) for (const registration of collector)
await this._setupFixtureForRegistration(registration, testInfo); await this._setupFixtureForRegistration(registration, testInfo);
}); });
@ -259,7 +261,7 @@ export class FixtureRunner {
// Do not run the function when fixture setup has already failed. // Do not run the function when fixture setup has already failed.
return null; return null;
} }
await testInfo._runAsStage({}, async () => { await testInfo._runAsStage({ title: 'run function', canTimeout: true }, async () => {
await fn(params, testInfo); await fn(params, testInfo);
}); });
} }

130
packages/playwright/src/worker/testInfo.ts
View file

@ -21,10 +21,10 @@ import type { TestInfoError, TestInfo, TestStatus, FullProject, FullConfig } fro
import type { AttachmentPayload, StepBeginPayload, StepEndPayload, WorkerInitParams } from '../common/ipc'; import type { AttachmentPayload, StepBeginPayload, StepEndPayload, WorkerInitParams } from '../common/ipc';
import type { TestCase } from '../common/test'; import type { TestCase } from '../common/test';
import { TimeoutManager } from './timeoutManager'; import { TimeoutManager } from './timeoutManager';
import type { RunnableType, TimeSlot } from './timeoutManager'; import type { RunnableDescription, RunnableType, TimeSlot } from './timeoutManager';
import type { Annotation, FullConfigInternal, FullProjectInternal } from '../common/config'; import type { Annotation, FullConfigInternal, FullProjectInternal } from '../common/config';
import type { Location } from '../../types/testReporter'; import type { Location } from '../../types/testReporter';
import { filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, serializeError, trimLongString } from '../util'; import { debugTest, filteredStackTrace, formatLocation, getContainedPath, normalizeAndSaveAttachment, serializeError, trimLongString } from '../util';
import { TestTracing } from './testTracing'; import { TestTracing } from './testTracing';
import type { Attachment } from './testTracing'; import type { Attachment } from './testTracing';
import type { StackFrame } from '@protocol/channels'; import type { StackFrame } from '@protocol/channels';
@ -49,19 +49,20 @@ export interface TestStepInternal {
} }
export type TestStage = { export type TestStage = {
title: string;
location?: Location;
stepCategory?: 'hook' | 'fixture';
runnableType?: RunnableType; runnableType?: RunnableType;
runnableSlot?: TimeSlot; runnableSlot?: TimeSlot;
stepInfo?: { canTimeout?: boolean;
title: string; allowSkip?: boolean;
category: 'hook' | 'fixture';
};
location?: Location;
allowAndStopOnSkips?: boolean;
stopOnChildError?: boolean; stopOnChildError?: boolean;
continueOnChildTimeout?: boolean;
step?: TestStepInternal; step?: TestStepInternal;
error?: Error; error?: Error;
triggeredSkip?: boolean; triggeredSkip?: boolean;
triggeredTimeout?: boolean;
}; };
export class TestInfoImpl implements TestInfo { export class TestInfoImpl implements TestInfo {
@ -234,22 +235,6 @@ export class TestInfoImpl implements TestInfo {
} }
} }
async _runWithTimeout(cb: () => Promise<any>): Promise<void> {
const timeoutError = await this._timeoutManager.runWithTimeout(cb);
// When interrupting, we arrive here with a timeoutError, but we should not
// consider it a timeout.
if (!this._wasInterrupted && timeoutError && !this._didTimeout) {
this._didTimeout = true;
const serialized = serializeError(timeoutError);
this.errors.push(serialized);
this._tracing.appendForError(serialized);
// Do not overwrite existing failure upon hook/teardown timeout.
if (this.status === 'passed' || this.status === 'skipped')
this.status = 'timedOut';
}
this.duration = this._timeoutManager.defaultSlotTimings().elapsed | 0;
}
private _findLastNonFinishedStep(filter: (step: TestStepInternal) => boolean) { private _findLastNonFinishedStep(filter: (step: TestStepInternal) => boolean) {
let result: TestStepInternal | undefined; let result: TestStepInternal | undefined;
const visit = (step: TestStepInternal) => { const visit = (step: TestStepInternal) => {
@ -367,6 +352,13 @@ export class TestInfoImpl implements TestInfo {
this.status = 'interrupted'; this.status = 'interrupted';
} }
_unhandledError(error: Error) {
this._failWithError(error, true /* isHardError */, true /* retriable */);
const stage = this._stages[this._stages.length - 1];
if (stage)
stage.error = stage.error ?? error;
}
_failWithError(error: Error, isHardError: boolean, retriable: boolean) { _failWithError(error: Error, isHardError: boolean, retriable: boolean) {
if (!retriable) if (!retriable)
this._hasNonRetriableError = true; this._hasNonRetriableError = true;
@ -378,12 +370,6 @@ export class TestInfoImpl implements TestInfo {
return; return;
if (isHardError) if (isHardError)
this._hasHardError = true; this._hasHardError = true;
const stage = this._stages[this._stages.length - 1];
if (stage) {
// Save the error to the stage here, so that unhandled rejections
// are attributed to the current stage. Among many errors, prefer the first one.
stage.error = stage.error ?? error;
}
if (this.status === 'passed' || this.status === 'skipped') if (this.status === 'passed' || this.status === 'skipped')
this.status = 'failed'; this.status = 'failed';
const serialized = serializeError(error); const serialized = serializeError(error);
@ -397,54 +383,88 @@ export class TestInfoImpl implements TestInfo {
async _runAsStage(stage: TestStage, cb: () => Promise<any>) { async _runAsStage(stage: TestStage, cb: () => Promise<any>) {
// Inherit some properties from parent. // Inherit some properties from parent.
const parent = this._stages[this._stages.length - 1]; const parent = this._stages[this._stages.length - 1];
stage.allowAndStopOnSkips = stage.allowAndStopOnSkips ?? parent?.allowAndStopOnSkips ?? false; stage.allowSkip = stage.allowSkip ?? parent?.allowSkip ?? false;
if (parent?.allowAndStopOnSkips && parent?.triggeredSkip) { if (parent?.allowSkip && parent?.triggeredSkip) {
// Do not run more child steps after "skip" has been triggered. // Do not run more child steps after "skip" has been triggered.
debugTest(`ignored stage "${stage.title}" after previous skip`);
return; return;
} }
if (parent?.stopOnChildError && parent?.error) { if (parent?.stopOnChildError && parent?.error) {
// Do not run more child steps after a previous one failed. // Do not run more child steps after a previous one failed.
debugTest(`ignored stage "${stage.title}" after previous error`);
return;
}
if (parent?.triggeredTimeout && !parent?.continueOnChildTimeout) {
// Do not run more child steps after a previous one timed out.
debugTest(`ignored stage "${stage.title}" after previous timeout`);
return; return;
} }
const runnable = stage.runnableType ? { if (debugTest.enabled) {
type: stage.runnableType, const location = stage.location ? ` at "${formatLocation(stage.location)}"` : ``;
slot: stage.runnableSlot, debugTest(`started stage "${stage.title}"${location}`);
location: stage.location, }
} : undefined; stage.step = stage.stepCategory ? this._addStep({ title: stage.title, category: stage.stepCategory, location: stage.location, wallTime: Date.now(), isStage: true }) : undefined;
this._stages.push(stage);
return await this._timeoutManager.withRunnable(runnable, async () => { let runnable: RunnableDescription | undefined;
stage.step = stage.stepInfo ? this._addStep({ ...stage.stepInfo, location: stage.location, wallTime: Date.now(), isStage: true }) : undefined; if (stage.canTimeout) {
this._stages.push(stage); // Choose the deepest runnable configuration.
runnable = { type: 'test' };
for (const s of this._stages) {
if (s.runnableType) {
runnable.type = s.runnableType;
runnable.location = s.location;
}
if (s.runnableSlot)
runnable.slot = s.runnableSlot;
}
}
const timeoutError = await this._timeoutManager.withRunnable(runnable, async () => {
try { try {
await cb(); await cb();
} catch (e) { } catch (e) {
if (stage.allowAndStopOnSkips && (e instanceof SkipError)) { if (stage.allowSkip && (e instanceof SkipError)) {
stage.triggeredSkip = true; stage.triggeredSkip = true;
if (this.status === 'passed') if (this.status === 'passed')
this.status = 'skipped'; this.status = 'skipped';
} else { } else {
// Prefer the first error.
stage.error = stage.error ?? e;
this._failWithError(e, true /* isHardError */, true /* retriable */); this._failWithError(e, true /* isHardError */, true /* retriable */);
} }
} }
if (parent) {
// Notify parent about child error and skip.
parent.error = parent.error ?? stage.error;
parent.triggeredSkip = parent.triggeredSkip || stage.triggeredSkip;
}
const index = this._stages.indexOf(stage);
if (index !== -1)
this._stages.splice(index, 1);
stage.step?.complete({ error: stage.error });
}); });
} if (timeoutError)
stage.triggeredTimeout = true;
_resetStages() { // When interrupting, we arrive here with a timeoutError, but we should not
this._stages.splice(0, this._stages.length); // consider it a timeout.
if (!this._wasInterrupted && !this._didTimeout && timeoutError) {
stage.error = stage.error ?? timeoutError;
this._didTimeout = true;
const serialized = serializeError(timeoutError);
this.errors.push(serialized);
this._tracing.appendForError(serialized);
// Do not overwrite existing failure upon hook/teardown timeout.
if (this.status === 'passed' || this.status === 'skipped')
this.status = 'timedOut';
}
if (parent) {
// Notify parent about child error, skip and timeout.
parent.error = parent.error ?? stage.error;
parent.triggeredSkip = parent.triggeredSkip || stage.triggeredSkip;
parent.triggeredTimeout = parent.triggeredTimeout || stage.triggeredTimeout;
}
if (this._stages[this._stages.length - 1] !== stage)
throw new Error(`Internal error: inconsistent stages!`);
this._stages.pop();
stage.step?.complete({ error: stage.error });
debugTest(`finished stage "${stage.title}"`);
} }
_isFailure() { _isFailure() {

24
packages/playwright/src/worker/timeoutManager.ts
View file

@ -56,16 +56,22 @@ export class TimeoutManager {
this._timeoutRunner.interrupt(); this._timeoutRunner.interrupt();
} }
async withRunnable<R>(runnable: RunnableDescription | undefined, cb: () => Promise<R>): Promise<R> { async withRunnable(runnable: RunnableDescription | undefined, cb: () => Promise<any>): Promise<Error | undefined> {
if (!runnable) if (!runnable) {
return await cb(); await cb();
return;
}
const existingRunnable = this._runnable; const existingRunnable = this._runnable;
const effectiveRunnable = { ...runnable }; const effectiveRunnable = { ...runnable };
if (!effectiveRunnable.slot) if (!effectiveRunnable.slot)
effectiveRunnable.slot = this._runnable.slot; effectiveRunnable.slot = this._runnable.slot;
this._updateRunnables(effectiveRunnable, undefined); this._updateRunnables(effectiveRunnable, undefined);
try { try {
return await cb(); await this._timeoutRunner.run(cb);
} catch (error) {
if (!(error instanceof TimeoutRunnerError))
throw error;
return this._createTimeoutError();
} finally { } finally {
this._updateRunnables(existingRunnable, undefined); this._updateRunnables(existingRunnable, undefined);
} }
@ -87,16 +93,6 @@ export class TimeoutManager {
this._timeoutRunner.updateTimeout(slot.timeout); this._timeoutRunner.updateTimeout(slot.timeout);
} }
async runWithTimeout(cb: () => Promise<any>): Promise<Error | undefined> {
try {
await this._timeoutRunner.run(cb);
} catch (error) {
if (!(error instanceof TimeoutRunnerError))
throw error;
return this._createTimeoutError();
}
}
setTimeout(timeout: number) { setTimeout(timeout: number) {
const slot = this._currentSlot(); const slot = this._currentSlot();
if (!slot.timeout) if (!slot.timeout)

196
packages/playwright/src/worker/workerMain.ts
View file

@ -15,7 +15,7 @@
*/ */
import { colors } from 'playwright-core/lib/utilsBundle'; import { colors } from 'playwright-core/lib/utilsBundle';
import { debugTest, formatLocation, relativeFilePath, serializeError } from '../util'; import { debugTest, relativeFilePath, serializeError } from '../util';
import { type TestBeginPayload, type TestEndPayload, type RunPayload, type DonePayload, type WorkerInitParams, type TeardownErrorsPayload, stdioChunkToParams } from '../common/ipc'; import { type TestBeginPayload, type TestEndPayload, type RunPayload, type DonePayload, type WorkerInitParams, type TeardownErrorsPayload, stdioChunkToParams } from '../common/ipc';
import { setCurrentTestInfo, setIsWorkerProcess } from '../common/globals'; import { setCurrentTestInfo, setIsWorkerProcess } from '../common/globals';
import { deserializeConfig } from '../common/configLoader'; import { deserializeConfig } from '../common/configLoader';
@ -23,7 +23,7 @@ import type { Suite, TestCase } from '../common/test';
import type { Annotation, FullConfigInternal, FullProjectInternal } from '../common/config'; import type { Annotation, FullConfigInternal, FullProjectInternal } from '../common/config';
import { FixtureRunner } from './fixtureRunner'; import { FixtureRunner } from './fixtureRunner';
import { ManualPromise, gracefullyCloseAll, removeFolders } from 'playwright-core/lib/utils'; import { ManualPromise, gracefullyCloseAll, removeFolders } from 'playwright-core/lib/utils';
import { TestInfoImpl, TestStage } from './testInfo'; import { TestInfoImpl, type TestStage } from './testInfo';
import { ProcessRunner } from '../common/process'; import { ProcessRunner } from '../common/process';
import { loadTestFile } from '../common/testLoader'; import { loadTestFile } from '../common/testLoader';
import { applyRepeatEachIndex, bindFileSuiteToProject, filterTestsRemoveEmptySuites } from '../common/suiteUtils'; import { applyRepeatEachIndex, bindFileSuiteToProject, filterTestsRemoveEmptySuites } from '../common/suiteUtils';
@ -146,11 +146,9 @@ export class WorkerMain extends ProcessRunner {
private async _teardownScopes() { private async _teardownScopes() {
// TODO: separate timeout for teardown? // TODO: separate timeout for teardown?
const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {}); const fakeTestInfo = new TestInfoImpl(this._config, this._project, this._params, undefined, 0, () => {}, () => {}, () => {});
await fakeTestInfo._runAsStage({ runnableType: 'teardown' }, async () => { await fakeTestInfo._runAsStage({ title: 'teardown scopes', runnableType: 'teardown' }, async () => {
await fakeTestInfo._runWithTimeout(async () => { await this._fixtureRunner.teardownScope('test', fakeTestInfo);
await this._fixtureRunner.teardownScope('test', fakeTestInfo); await this._fixtureRunner.teardownScope('worker', fakeTestInfo);
await this._fixtureRunner.teardownScope('worker', fakeTestInfo);
});
}); });
this._fatalErrors.push(...fakeTestInfo.errors); this._fatalErrors.push(...fakeTestInfo.errors);
} }
@ -168,7 +166,7 @@ export class WorkerMain extends ProcessRunner {
// and unhandled errors - both lead to the test failing. This is good for regular tests, // and unhandled errors - both lead to the test failing. This is good for regular tests,
// so that you can, e.g. expect() from inside an event handler. The test fails, // so that you can, e.g. expect() from inside an event handler. The test fails,
// and we restart the worker. // and we restart the worker.
this._currentTest._failWithError(error, true /* isHardError */, true /* retriable */); this._currentTest._unhandledError(error);
// For tests marked with test.fail(), this might be a problem when unhandled error // For tests marked with test.fail(), this might be a problem when unhandled error
// is not coming from the user test code (legit failure), but from fixtures or test runner. // is not coming from the user test code (legit failure), but from fixtures or test runner.
@ -308,76 +306,74 @@ export class WorkerMain extends ProcessRunner {
this._lastRunningTests.shift(); this._lastRunningTests.shift();
let shouldRunAfterEachHooks = false; let shouldRunAfterEachHooks = false;
await testInfo._runWithTimeout(async () => { await testInfo._runAsStage({ title: 'setup and test', runnableType: 'test', allowSkip: true, stopOnChildError: true }, async () => {
await testInfo._runAsStage({ allowAndStopOnSkips: true, stopOnChildError: true }, async () => { await testInfo._runAsStage({ title: 'start tracing', canTimeout: true }, async () => {
await testInfo._runAsStage({}, async () => { // Ideally, "trace" would be an config-level option belonging to the
// Ideally, "trace" would be an config-level option belonging to the // test runner instead of a fixture belonging to Playwright.
// test runner instead of a fixture belonging to Playwright. // However, for backwards compatibility, we have to read it from a fixture today.
// However, for backwards compatibility, we have to read it from a fixture today. // We decided to not introduce the config-level option just yet.
// We decided to not introduce the config-level option just yet. const traceFixtureRegistration = test._pool!.resolve('trace');
const traceFixtureRegistration = test._pool!.resolve('trace'); if (!traceFixtureRegistration)
if (!traceFixtureRegistration)
return;
if (typeof traceFixtureRegistration.fn === 'function')
throw new Error(`"trace" option cannot be a function`);
await testInfo._tracing.startIfNeeded(traceFixtureRegistration.fn);
});
if (this._isStopped || isSkipped) {
// Two reasons to get here:
// - Last test is skipped, so we should not run the test, but run the cleanup.
// - Worker is requested to stop, but was not able to run full cleanup yet.
// We should skip the test, but run the cleanup.
testInfo.status = 'skipped';
return; return;
} if (typeof traceFixtureRegistration.fn === 'function')
throw new Error(`"trace" option cannot be a function`);
await testInfo._tracing.startIfNeeded(traceFixtureRegistration.fn);
});
await removeFolders([testInfo.outputDir]); if (this._isStopped || isSkipped) {
// Two reasons to get here:
// - Last test is skipped, so we should not run the test, but run the cleanup.
// - Worker is requested to stop, but was not able to run full cleanup yet.
// We should skip the test, but run the cleanup.
testInfo.status = 'skipped';
return;
}
let testFunctionParams: object | null = null; await removeFolders([testInfo.outputDir]);
const beforeHooksStage: TestStage = { stepInfo: { category: 'hook', title: 'Before Hooks' }, stopOnChildError: true };
await testInfo._runAsStage(beforeHooksStage, async () => {
// Run "beforeAll" hooks, unless already run during previous tests.
for (const suite of suites)
await this._runBeforeAllHooksForSuite(suite, testInfo);
// Run "beforeEach" hooks. Once started with "beforeEach", we must run all "afterEach" hooks as well. let testFunctionParams: object | null = null;
shouldRunAfterEachHooks = !beforeHooksStage.error && !beforeHooksStage.triggeredSkip; const beforeHooksStage: TestStage = { title: 'Before Hooks', stepCategory: 'hook', stopOnChildError: true };
await this._runEachHooksForSuites(suites, 'beforeEach', testInfo); await testInfo._runAsStage(beforeHooksStage, async () => {
// Run "beforeAll" hooks, unless already run during previous tests.
for (const suite of suites)
await this._runBeforeAllHooksForSuite(suite, testInfo);
// Setup fixtures required by the test. // Run "beforeEach" hooks. Once started with "beforeEach", we must run all "afterEach" hooks as well.
testFunctionParams = await this._fixtureRunner.resolveParametersForFunction(test.fn, testInfo, 'test'); shouldRunAfterEachHooks = !beforeHooksStage.error && !beforeHooksStage.triggeredSkip && !beforeHooksStage.triggeredTimeout;
}); await this._runEachHooksForSuites(suites, 'beforeEach', testInfo);
if (testFunctionParams === null) { // Setup fixtures required by the test.
// Fixture setup failed or was skipped, we should not run the test now. testFunctionParams = await this._fixtureRunner.resolveParametersForFunction(test.fn, testInfo, 'test');
return; });
}
await testInfo._runAsStage({}, async () => { if (testFunctionParams === null) {
// Now run the test itself. // Fixture setup failed or was skipped, we should not run the test now.
debugTest(`test function started`); return;
const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]"). }
await fn(testFunctionParams, testInfo);
debugTest(`test function finished`); await testInfo._runAsStage({ title: 'test function', canTimeout: true }, async () => {
}); // Now run the test itself.
const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]").
await fn(testFunctionParams, testInfo);
}); });
}); });
// Reset stages after a possible timeout. Note: we can remove this if we race each stage against the timeout. // Update duration, so it is available in fixture teardown and afterEach hooks.
testInfo._resetStages(); testInfo.duration = testInfo._timeoutManager.defaultSlotTimings().elapsed | 0;
// A timed-out test gets a full additional timeout to run after hooks. // A timed-out test gets a full additional timeout to run after hooks.
const afterHooksSlot = testInfo._didTimeout ? { timeout: this._project.project.timeout, elapsed: 0 } : undefined; const afterHooksSlot = testInfo._didTimeout ? { timeout: this._project.project.timeout, elapsed: 0 } : undefined;
await testInfo._runAsStage({ stepInfo: { category: 'hook', title: 'After Hooks' }, runnableType: 'afterHooks', runnableSlot: afterHooksSlot }, async () => { await testInfo._runAsStage({
await testInfo._runWithTimeout(async () => { title: 'After Hooks',
// Note: do not wrap all teardown steps together, because failure in any of them stepCategory: 'hook',
// does not prevent further teardown steps from running. runnableType: 'afterHooks',
runnableSlot: afterHooksSlot,
continueOnChildTimeout: true, // Make sure the full cleanup still runs after regular cleanup timeout.
}, async () => {
// Wrap cleanup steps in a stage, to stop running after one of them times out.
await testInfo._runAsStage({ title: 'regular cleanup' }, async () => {
// Run "immediately upon test function finish" callback. // Run "immediately upon test function finish" callback.
debugTest(`on-test-function-finish callback started`); await testInfo._runAsStage({ title: 'on-test-function-finish', canTimeout: true }, async () => testInfo._onDidFinishTestFunction?.());
await testInfo._runAsStage({}, async () => testInfo._onDidFinishTestFunction?.());
debugTest(`on-test-function-finish callback finished`);
// Run "afterEach" hooks, unless we failed at beforeAll stage. // Run "afterEach" hooks, unless we failed at beforeAll stage.
if (shouldRunAfterEachHooks) if (shouldRunAfterEachHooks)
@ -385,17 +381,20 @@ export class WorkerMain extends ProcessRunner {
// Teardown test-scoped fixtures. Attribute to 'test' so that users understand // Teardown test-scoped fixtures. Attribute to 'test' so that users understand
// they should probably increase the test timeout to fix this issue. // they should probably increase the test timeout to fix this issue.
debugTest(`tearing down test scope started`); await testInfo._runAsStage({ title: 'teardown test scope', runnableType: 'test' }, async () => {
await this._fixtureRunner.teardownScope('test', testInfo); await this._fixtureRunner.teardownScope('test', testInfo);
debugTest(`tearing down test scope finished`); });
// Run "afterAll" hooks for suites that are not shared with the next test. // Run "afterAll" hooks for suites that are not shared with the next test.
// In case of failure the worker will be stopped and we have to make sure that afterAll // In case of failure the worker will be stopped and we have to make sure that afterAll
// hooks run before worker fixtures teardown. // hooks run before worker fixtures teardown.
for (const suite of reversedSuites) { // Continue running "afterAll" hooks even after some of them timeout.
if (!nextSuites.has(suite) || testInfo._isFailure()) await testInfo._runAsStage({ title: `after hooks suites`, continueOnChildTimeout: true }, async () => {
await this._runAfterAllHooksForSuite(suite, testInfo); for (const suite of reversedSuites) {
} if (!nextSuites.has(suite) || testInfo._isFailure())
await this._runAfterAllHooksForSuite(suite, testInfo);
}
});
}); });
if (testInfo._isFailure()) if (testInfo._isFailure())
@ -407,32 +406,28 @@ export class WorkerMain extends ProcessRunner {
this._didRunFullCleanup = true; this._didRunFullCleanup = true;
// Give it more time for the full cleanup. // Give it more time for the full cleanup.
await testInfo._runWithTimeout(async () => { const teardownSlot = { timeout: this._project.project.timeout, elapsed: 0 };
debugTest(`running full cleanup after the failure`); await testInfo._runAsStage({ title: 'full cleanup', runnableType: 'teardown', runnableSlot: teardownSlot }, async () => {
// Attribute to 'test' so that users understand they should probably increate the test timeout to fix this issue.
const teardownSlot = { timeout: this._project.project.timeout, elapsed: 0 }; await testInfo._runAsStage({ title: 'teardown test scope', runnableType: 'test' }, async () => {
await testInfo._runAsStage({ runnableType: 'teardown', runnableSlot: teardownSlot }, async () => {
// Attribute to 'test' so that users understand they should probably increate the test timeout to fix this issue.
debugTest(`tearing down test scope started`);
await this._fixtureRunner.teardownScope('test', testInfo); await this._fixtureRunner.teardownScope('test', testInfo);
debugTest(`tearing down test scope finished`);
for (const suite of reversedSuites)
await this._runAfterAllHooksForSuite(suite, testInfo);
// Attribute to 'teardown' because worker fixtures are not perceived as a part of a test.
debugTest(`tearing down worker scope started`);
await this._fixtureRunner.teardownScope('worker', testInfo);
debugTest(`tearing down worker scope finished`);
}); });
for (const suite of reversedSuites)
await this._runAfterAllHooksForSuite(suite, testInfo);
// Attribute to 'teardown' because worker fixtures are not perceived as a part of a test.
await this._fixtureRunner.teardownScope('worker', testInfo);
}); });
} }
}); });
await testInfo._runAsStage({}, async () => { await testInfo._runAsStage({ title: 'stop tracing' }, async () => {
await testInfo._tracing.stopIfNeeded(); await testInfo._tracing.stopIfNeeded();
}); });
testInfo.duration = testInfo._timeoutManager.defaultSlotTimings().elapsed | 0;
this._currentTest = null; this._currentTest = null;
setCurrentTestInfo(null); setCurrentTestInfo(null);
this.dispatchEvent('testEnd', buildTestEndPayload(testInfo)); this.dispatchEvent('testEnd', buildTestEndPayload(testInfo));
@ -469,24 +464,25 @@ export class WorkerMain extends ProcessRunner {
private async _runBeforeAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl) { private async _runBeforeAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl) {
if (this._activeSuites.has(suite)) if (this._activeSuites.has(suite))
return 'success'; return;
const extraAnnotations: Annotation[] = []; const extraAnnotations: Annotation[] = [];
this._activeSuites.set(suite, extraAnnotations); this._activeSuites.set(suite, extraAnnotations);
return await this._runAllHooksForSuite(suite, testInfo, 'beforeAll', extraAnnotations); await this._runAllHooksForSuite(suite, testInfo, 'beforeAll', extraAnnotations);
} }
private async _runAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl, type: 'beforeAll' | 'afterAll', extraAnnotations?: Annotation[]) { private async _runAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl, type: 'beforeAll' | 'afterAll', extraAnnotations?: Annotation[]) {
// Always run all the hooks, and capture the first error. // Always run all the hooks, and capture the first error.
await testInfo._runAsStage({}, async () => { await testInfo._runAsStage({ title: `${type} hooks`, continueOnChildTimeout: true }, async () => {
for (const hook of this._collectHooksAndModifiers(suite, type, testInfo)) { for (const hook of this._collectHooksAndModifiers(suite, type, testInfo)) {
debugTest(`${hook.type} hook at "${formatLocation(hook.location)}" started`);
// Separate time slot for each beforeAll/afterAll hook. // Separate time slot for each beforeAll/afterAll hook.
const timeSlot = { timeout: this._project.project.timeout, elapsed: 0 }; const timeSlot = { timeout: this._project.project.timeout, elapsed: 0 };
const stage: TestStage = { const stage: TestStage = {
title: hook.title,
runnableType: hook.type, runnableType: hook.type,
runnableSlot: timeSlot, runnableSlot: timeSlot,
stepInfo: { category: 'hook', title: hook.title }, stepCategory: 'hook',
location: hook.location, location: hook.location,
continueOnChildTimeout: true, // Make sure to teardown the scope even after hook timeout.
}; };
await testInfo._runAsStage(stage, async () => { await testInfo._runAsStage(stage, async () => {
const existingAnnotations = new Set(testInfo.annotations); const existingAnnotations = new Set(testInfo.annotations);
@ -500,33 +496,33 @@ export class WorkerMain extends ProcessRunner {
// Note: we must teardown even after hook fails, because we'll run more hooks. // Note: we must teardown even after hook fails, because we'll run more hooks.
await this._fixtureRunner.teardownScope('test', testInfo); await this._fixtureRunner.teardownScope('test', testInfo);
}); });
if (stage.error && !this._skipRemainingTestsInSuite) { if ((stage.error || stage.triggeredTimeout) && type === 'beforeAll' && !this._skipRemainingTestsInSuite) {
// This will inform dispatcher that we should not run more tests from this group // This will inform dispatcher that we should not run more tests from this group
// because we had a beforeAll error. // because we had a beforeAll error.
// This behavior avoids getting the same common error for each test. // This behavior avoids getting the same common error for each test.
this._skipRemainingTestsInSuite = suite; this._skipRemainingTestsInSuite = suite;
} }
debugTest(`${hook.type} hook at "${formatLocation(hook.location)}" finished`);
} }
}); });
} }
private async _runAfterAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl) { private async _runAfterAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl) {
if (!this._activeSuites.has(suite)) if (!this._activeSuites.has(suite))
return 'success'; return;
this._activeSuites.delete(suite); this._activeSuites.delete(suite);
return await this._runAllHooksForSuite(suite, testInfo, 'afterAll'); await this._runAllHooksForSuite(suite, testInfo, 'afterAll');
} }
private async _runEachHooksForSuites(suites: Suite[], type: 'beforeEach' | 'afterEach', testInfo: TestInfoImpl) { private async _runEachHooksForSuites(suites: Suite[], type: 'beforeEach' | 'afterEach', testInfo: TestInfoImpl) {
// Always run all the hooks, and capture the first error. // Wrap hooks in a stage, to always run all of them and capture the first error.
await testInfo._runAsStage({}, async () => { await testInfo._runAsStage({ title: `${type} hooks` }, async () => {
const hooks = suites.map(suite => this._collectHooksAndModifiers(suite, type, testInfo)).flat(); const hooks = suites.map(suite => this._collectHooksAndModifiers(suite, type, testInfo)).flat();
for (const hook of hooks) { for (const hook of hooks) {
await testInfo._runAsStage({ await testInfo._runAsStage({
title: hook.title,
runnableType: hook.type, runnableType: hook.type,
location: hook.location, location: hook.location,
stepInfo: { category: 'hook', title: hook.title }, stepCategory: 'hook',
}, async () => { }, async () => {
await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo, 'test'); await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo, 'test');
}); });

2
tests/playwright-test/basic.spec.ts
View file

@ -559,7 +559,7 @@ test('should not allow mixing test types', async ({ runInlineTest }) => {
export const test2 = test.extend({ export const test2 = test.extend({
value: 42, value: 42,
}); });
test.describe("test1 suite", () => { test.describe("test1 suite", () => {
test2("test 2", async () => {}); test2("test 2", async () => {});
}); });

24
tests/playwright-test/timeout.spec.ts
View file

@ -455,3 +455,27 @@ test('should respect test.describe.configure', async ({ runInlineTest }) => {
expect(result.output).toContain('test1-1000'); expect(result.output).toContain('test1-1000');
expect(result.output).toContain('test2-2000'); expect(result.output).toContain('test2-2000');
}); });
test('beforeEach timeout should prevent others from running', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test.beforeEach(async () => {
console.log('\\n%%beforeEach1');
await new Promise(f => setTimeout(f, 2500));
});
test.beforeEach(async () => {
console.log('\\n%%beforeEach2');
});
test('test', async ({}) => {
});
test.afterEach(async () => {
console.log('\\n%%afterEach');
await new Promise(f => setTimeout(f, 1500));
});
`
}, { timeout: 2000 });
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.outputLines).toEqual(['beforeEach1', 'afterEach']);
});