diff --git a/packages/playwright-core/src/utils/async.ts b/packages/playwright-core/src/utils/async.ts index 2648fcaac8..e91702cad6 100644 --- a/packages/playwright-core/src/utils/async.ts +++ b/packages/playwright-core/src/utils/async.ts @@ -19,7 +19,7 @@ import { monotonicTime } from './utils'; export class TimeoutRunnerError extends Error {} type TimeoutRunnerData = { - lastElapsedSync: number, + start: number, timer: NodeJS.Timer | undefined, timeoutPromise: ManualPromise, }; @@ -35,7 +35,7 @@ export class TimeoutRunner { async run(cb: () => Promise): Promise { const running = this._running = { - lastElapsedSync: monotonicTime(), + start: monotonicTime(), timer: undefined, timeoutPromise: new ManualPromise(), }; @@ -47,6 +47,7 @@ export class TimeoutRunner { this._updateTimeout(running, this._timeout); return await resultPromise; } finally { + this._elapsed += monotonicTime() - running.start; this._updateTimeout(running, 0); if (this._running === running) this._running = undefined; @@ -58,27 +59,15 @@ export class TimeoutRunner { this._updateTimeout(this._running, -1); } - elapsed() { - this._syncElapsedAndStart(); - return this._elapsed; - } - - updateTimeout(timeout: number, elapsed?: number) { + updateTimeout(timeout: number) { this._timeout = timeout; - if (elapsed !== undefined) { - this._syncElapsedAndStart(); - this._elapsed = elapsed; - } if (this._running) this._updateTimeout(this._running, timeout); } - private _syncElapsedAndStart() { - if (this._running) { - const now = monotonicTime(); - this._elapsed += now - this._running.lastElapsedSync; - this._running.lastElapsedSync = now; - } + resetTimeout(timeout: number) { + this._elapsed = 0; + this.updateTimeout(timeout); } private _updateTimeout(running: TimeoutRunnerData, timeout: number) { @@ -86,10 +75,10 @@ export class TimeoutRunner { clearTimeout(running.timer); running.timer = undefined; } - this._syncElapsedAndStart(); if (timeout === 0) return; - timeout = timeout - this._elapsed; + const elapsed = (monotonicTime() - running.start) + this._elapsed; + timeout = timeout - elapsed; if (timeout <= 0) running.timeoutPromise.reject(new TimeoutRunnerError()); else diff --git a/packages/playwright-test/src/dispatcher.ts b/packages/playwright-test/src/dispatcher.ts index a7fa3433c7..7002130a63 100644 --- a/packages/playwright-test/src/dispatcher.ts +++ b/packages/playwright-test/src/dispatcher.ts @@ -59,8 +59,15 @@ export class Dispatcher { this._queue = testGroups; for (const group of testGroups) { this._queueHashCount.set(group.workerHash, 1 + (this._queueHashCount.get(group.workerHash) || 0)); - for (const test of group.tests) + for (const test of group.tests) { this._testById.set(test._id, { test, resultByWorkerIndex: new Map() }); + for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) { + for (const hook of suite.hooks) { + if (!this._testById.has(hook._id)) + this._testById.set(hook._id, { test: hook, resultByWorkerIndex: new Map() }); + } + } + } } } @@ -177,20 +184,25 @@ export class Dispatcher { const remainingByTestId = new Map(testGroup.tests.map(e => [ e._id, e ])); const failedTestIds = new Set(); + let runningHookId: string | undefined; const onTestBegin = (params: TestBeginPayload) => { const data = this._testById.get(params.testId)!; + if (data.test._type !== 'test') + runningHookId = params.testId; if (this._hasReachedMaxFailures()) return; const result = data.test._appendTestResult(); data.resultByWorkerIndex.set(worker.workerIndex, { result, stepStack: new Set(), steps: new Map() }); result.workerIndex = worker.workerIndex; result.startTime = new Date(params.startWallTime); - this._reporter.onTestBegin?.(data.test, result); + if (data.test._type === 'test') + this._reporter.onTestBegin?.(data.test, result); }; worker.addListener('testBegin', onTestBegin); const onTestEnd = (params: TestEndPayload) => { + runningHookId = undefined; remainingByTestId.delete(params.testId); if (this._hasReachedMaxFailures()) return; @@ -212,7 +224,7 @@ export class Dispatcher { test.annotations = params.annotations; test.timeout = params.timeout; const isFailure = result.status !== 'skipped' && result.status !== test.expectedStatus; - if (isFailure) + if (isFailure && test._type === 'test') failedTestIds.add(params.testId); this._reportTestEnd(test, result); }; @@ -278,7 +290,7 @@ export class Dispatcher { // - there are no remaining // - we are here not because something failed // - no unrecoverable worker error - if (!remaining.length && !failedTestIds.size && !params.fatalErrors.length && !params.skipRemaining) { + if (!remaining.length && !failedTestIds.size && !params.fatalErrors.length) { if (this._isWorkerRedundant(worker)) worker.stop(); doneWithJob(); @@ -290,8 +302,18 @@ export class Dispatcher { // In case of fatal error, report first remaining test as failing with this error, // and all others as skipped. - if (params.fatalErrors.length || params.skipRemaining) { - let shouldAddFatalErrorsToNextTest = params.fatalErrors.length > 0; + if (params.fatalErrors.length) { + // Perhaps we were running a hook - report it as failed. + if (runningHookId) { + const data = this._testById.get(runningHookId)!; + const { result } = data.resultByWorkerIndex.get(worker.workerIndex)!; + result.errors = [...params.fatalErrors]; + result.error = result.errors[0]; + result.status = 'failed'; + this._reporter.onTestEnd?.(data.test, result); + } + + let first = true; for (const test of remaining) { if (this._hasReachedMaxFailures()) break; @@ -303,23 +325,24 @@ export class Dispatcher { result = runData.result; } else { result = data.test._appendTestResult(); - this._reporter.onTestBegin?.(test, result); + if (test._type === 'test') + this._reporter.onTestBegin?.(test, result); } - result.errors = shouldAddFatalErrorsToNextTest ? [...params.fatalErrors] : []; + result.errors = [...params.fatalErrors]; result.error = result.errors[0]; - result.status = shouldAddFatalErrorsToNextTest ? 'failed' : 'skipped'; + result.status = first ? 'failed' : 'skipped'; this._reportTestEnd(test, result); failedTestIds.add(test._id); - shouldAddFatalErrorsToNextTest = false; + first = false; } - if (shouldAddFatalErrorsToNextTest) { + if (first) { // We had a fatal error after all tests have passed - most likely in the afterAll hook. // Let's just fail the test run. this._hasWorkerErrors = true; for (const error of params.fatalErrors) this._reporter.onError?.(error); } - // Since we pretend that all remaining tests failed/skipped, there is nothing else to run, + // Since we pretend that all remaining tests failed, there is nothing else to run, // except for possible retries. remaining = []; } @@ -352,7 +375,8 @@ export class Dispatcher { // Emulate a "skipped" run, and drop this test from remaining. const result = test._appendTestResult(); - this._reporter.onTestBegin?.(test, result); + if (test._type === 'test') + this._reporter.onTestBegin?.(test, result); result.status = 'skipped'; this._reportTestEnd(test, result); return false; @@ -384,7 +408,7 @@ export class Dispatcher { worker.on('done', onDone); const onExit = (expectedly: boolean) => { - onDone({ skipRemaining: false, fatalErrors: expectedly ? [] : [{ value: 'Worker process exited unexpectedly' }] }); + onDone({ fatalErrors: expectedly ? [] : [{ value: 'Worker process exited unexpectedly' }] }); }; worker.on('exit', onExit); @@ -436,9 +460,10 @@ export class Dispatcher { } private _reportTestEnd(test: TestCase, result: TestResult) { - if (result.status !== 'skipped' && result.status !== test.expectedStatus) + if (test._type === 'test' && result.status !== 'skipped' && result.status !== test.expectedStatus) ++this._failureCount; - this._reporter.onTestEnd?.(test, result); + if (test._type === 'test') + this._reporter.onTestEnd?.(test, result); const maxFailures = this._loader.fullConfig().maxFailures; if (maxFailures && this._failureCount === maxFailures) this.stop().catch(e => {}); diff --git a/packages/playwright-test/src/fixtures.ts b/packages/playwright-test/src/fixtures.ts index 2bb6462711..f0886bbd8d 100644 --- a/packages/playwright-test/src/fixtures.ts +++ b/packages/playwright-test/src/fixtures.ts @@ -51,7 +51,7 @@ class Fixture { this.value = null; } - async setup(testInfo: TestInfo) { + async setup(workerInfo: WorkerInfo, testInfo: TestInfo | undefined) { if (typeof this.registration.fn !== 'function') { this.value = this.registration.fn; return; @@ -60,7 +60,7 @@ class Fixture { const params: { [key: string]: any } = {}; for (const name of this.registration.deps) { const registration = this.runner.pool!.resolveDependency(this.registration, name)!; - const dep = await this.runner.setupFixtureForRegistration(registration, testInfo); + const dep = await this.runner.setupFixtureForRegistration(registration, workerInfo, testInfo); dep.usages.add(this); params[name] = dep.value; } @@ -77,7 +77,6 @@ class Fixture { useFuncStarted.resolve(); await this._useFuncFinished; }; - const workerInfo: WorkerInfo = { config: testInfo.config, parallelIndex: testInfo.parallelIndex, workerIndex: testInfo.workerIndex, project: testInfo.project }; const info = this.registration.scope === 'worker' ? workerInfo : testInfo; this._selfTeardownComplete = Promise.resolve().then(() => this.registration.fn(params, useFunc, info)).catch((e: any) => { if (!useFuncStarted.isDone()) @@ -262,12 +261,12 @@ export class FixtureRunner { throw error; } - async resolveParametersForFunction(fn: Function, testInfo: TestInfo): Promise { + async resolveParametersForFunction(fn: Function, workerInfo: WorkerInfo, testInfo: TestInfo | undefined): Promise { // Install all automatic fixtures. for (const registration of this.pool!.registrations.values()) { const shouldSkip = !testInfo && registration.scope === 'test'; if (registration.auto && !shouldSkip) - await this.setupFixtureForRegistration(registration, testInfo); + await this.setupFixtureForRegistration(registration, workerInfo, testInfo); } // Install used fixtures. @@ -275,18 +274,18 @@ export class FixtureRunner { const params: { [key: string]: any } = {}; for (const name of names) { const registration = this.pool!.registrations.get(name)!; - const fixture = await this.setupFixtureForRegistration(registration, testInfo); + const fixture = await this.setupFixtureForRegistration(registration, workerInfo, testInfo); params[name] = fixture.value; } return params; } - async resolveParametersAndRunFunction(fn: Function, testInfo: TestInfo) { - const params = await this.resolveParametersForFunction(fn, testInfo); - return fn(params, testInfo); + async resolveParametersAndRunFunction(fn: Function, workerInfo: WorkerInfo, testInfo: TestInfo | undefined) { + const params = await this.resolveParametersForFunction(fn, workerInfo, testInfo); + return fn(params, testInfo || workerInfo); } - async setupFixtureForRegistration(registration: FixtureRegistration, testInfo: TestInfo): Promise { + async setupFixtureForRegistration(registration: FixtureRegistration, workerInfo: WorkerInfo, testInfo: TestInfo | undefined): Promise { if (registration.scope === 'test') this.testScopeClean = false; @@ -296,7 +295,7 @@ export class FixtureRunner { fixture = new Fixture(this, registration); this.instanceForId.set(registration.id, fixture); - await fixture.setup(testInfo); + await fixture.setup(workerInfo, testInfo); return fixture; } diff --git a/packages/playwright-test/src/index.ts b/packages/playwright-test/src/index.ts index e563b08d61..eb8f46b976 100644 --- a/packages/playwright-test/src/index.ts +++ b/packages/playwright-test/src/index.ts @@ -423,7 +423,7 @@ export const test = _baseTest.extend({ })); // 7. Cleanup created contexts when we know it's safe - this will produce nice error message. - if (testInfo.status === 'timedOut' && testInfo.errors.some(error => error.message?.match(/Timeout of \d+ms exceeded in beforeAll hook./))) { + if (hookType(testInfo) === 'beforeAll' && testInfo.status === 'timedOut') { const anyContext = leftoverContexts[0]; const pendingCalls = anyContext ? formatPendingCalls((anyContext as any)._connection.pendingProtocolCalls()) : ''; await Promise.all(leftoverContexts.filter(c => createdContexts.has(c)).map(c => c.close())); @@ -519,9 +519,9 @@ function formatStackFrame(frame: StackFrame) { } function hookType(testInfo: TestInfo): 'beforeAll' | 'afterAll' | undefined { - if ((testInfo as any)._currentRunnable?.type === 'beforeAll') + if (testInfo.title.startsWith('beforeAll')) return 'beforeAll'; - if ((testInfo as any)._currentRunnable?.type === 'afterAll') + if (testInfo.title.startsWith('afterAll')) return 'afterAll'; } diff --git a/packages/playwright-test/src/ipc.ts b/packages/playwright-test/src/ipc.ts index 91d49ab332..55e1830c0b 100644 --- a/packages/playwright-test/src/ipc.ts +++ b/packages/playwright-test/src/ipc.ts @@ -76,7 +76,6 @@ export type RunPayload = { export type DonePayload = { fatalErrors: TestError[]; - skipRemaining: boolean; }; export type TestOutputPayload = { diff --git a/packages/playwright-test/src/project.ts b/packages/playwright-test/src/project.ts index 8de0b8486b..50df921218 100644 --- a/packages/playwright-test/src/project.ts +++ b/packages/playwright-test/src/project.ts @@ -18,7 +18,6 @@ import type { FullProject, Fixtures, FixturesWithLocation } from './types'; import { Suite, TestCase } from './test'; import { FixturePool, isFixtureOption } from './fixtures'; import { TestTypeImpl } from './testType'; -import { calculateSha1 } from 'playwright-core/lib/utils/utils'; export class ProjectImpl { config: FullProject; @@ -53,8 +52,10 @@ export class ProjectImpl { for (const parent of parents) { if (parent._use.length) pool = new FixturePool(parent._use, pool, parent._isDescribe); - for (const hook of parent._hooks) + for (const hook of parent._eachHooks) pool.validateFunction(hook.fn, hook.type + ' hook', hook.location); + for (const hook of parent.hooks) + pool.validateFunction(hook.fn, hook._type + ' hook', hook.location); for (const modifier of parent._modifiers) pool.validateFunction(modifier.fn, modifier.type + ' modifier', modifier.location); } @@ -65,21 +66,19 @@ export class ProjectImpl { return this.testPools.get(test)!; } - private _cloneEntries(from: Suite, to: Suite, repeatEachIndex: number, filter: (test: TestCase) => boolean, relativeTitlePath: string): boolean { + private _cloneEntries(from: Suite, to: Suite, repeatEachIndex: number, filter: (test: TestCase) => boolean): boolean { for (const entry of from._entries) { if (entry instanceof Suite) { const suite = entry._clone(); to._addSuite(suite); - if (!this._cloneEntries(entry, suite, repeatEachIndex, filter, relativeTitlePath + ' ' + suite.title)) { + if (!this._cloneEntries(entry, suite, repeatEachIndex, filter)) { to._entries.pop(); to.suites.pop(); } } else { const test = entry._clone(); test.retries = this.config.retries; - // We rely upon relative paths being unique. - // See `getClashingTestsPerSuite()` in `runner.ts`. - test._id = `${calculateSha1(relativeTitlePath + ' ' + entry.title)}@${entry._requireFile}#run${this.index}-repeat${repeatEachIndex}`; + test._id = `${entry._ordinalInFile}@${entry._requireFile}#run${this.index}-repeat${repeatEachIndex}`; test.repeatEachIndex = repeatEachIndex; test._projectIndex = this.index; to._addTest(test); @@ -95,12 +94,21 @@ export class ProjectImpl { } if (!to._entries.length) return false; + for (const hook of from.hooks) { + const clone = hook._clone(); + clone.retries = 1; + clone._pool = this.buildPool(hook); + clone._projectIndex = this.index; + clone._id = `${hook._ordinalInFile}@${hook._requireFile}#run${this.index}-repeat${repeatEachIndex}`; + clone.repeatEachIndex = repeatEachIndex; + to._addAllHook(clone); + } return true; } cloneFileSuite(suite: Suite, repeatEachIndex: number, filter: (test: TestCase) => boolean): Suite | undefined { const result = suite._clone(); - return this._cloneEntries(suite, result, repeatEachIndex, filter, '') ? result : undefined; + return this._cloneEntries(suite, result, repeatEachIndex, filter) ? result : undefined; } private resolveFixtures(testType: TestTypeImpl, configUse: Fixtures): FixturesWithLocation[] { diff --git a/packages/playwright-test/src/reporters/line.ts b/packages/playwright-test/src/reporters/line.ts index f5d1083f4c..4b089e2232 100644 --- a/packages/playwright-test/src/reporters/line.ts +++ b/packages/playwright-test/src/reporters/line.ts @@ -61,7 +61,8 @@ class LineReporter extends BaseReporter { override onTestEnd(test: TestCase, result: TestResult) { super.onTestEnd(test, result); - ++this._current; + if (!test.title.startsWith('beforeAll') && !test.title.startsWith('afterAll')) + ++this._current; const retriesSuffix = this.totalTestCount < this._current ? ` (retries)` : ``; const title = `[${this._current}/${this.totalTestCount}]${retriesSuffix} ${formatTestTitle(this.config, test)}`; const suffix = result.retry ? ` (retry #${result.retry})` : ''; diff --git a/packages/playwright-test/src/test.ts b/packages/playwright-test/src/test.ts index fcba982911..121b1d6f7b 100644 --- a/packages/playwright-test/src/test.ts +++ b/packages/playwright-test/src/test.ts @@ -17,7 +17,7 @@ import type { FixturePool } from './fixtures'; import * as reporterTypes from '../types/testReporter'; import type { TestTypeImpl } from './testType'; -import { Annotation, FixturesWithLocation, Location } from './types'; +import { Annotations, FixturesWithLocation, Location, TestCaseType } from './types'; import { FullProject } from './types'; class Base { @@ -45,9 +45,10 @@ export class Suite extends Base implements reporterTypes.Suite { _use: FixturesWithLocation[] = []; _isDescribe = false; _entries: (Suite | TestCase)[] = []; - _hooks: { type: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll', fn: Function, location: Location }[] = []; + hooks: TestCase[] = []; + _eachHooks: { type: 'beforeEach' | 'afterEach', fn: Function, location: Location }[] = []; _timeout: number | undefined; - _annotations: Annotation[] = []; + _annotations: Annotations = []; _modifiers: Modifier[] = []; _parallelMode: 'default' | 'serial' | 'parallel' = 'default'; _projectConfig: FullProject | undefined; @@ -65,6 +66,11 @@ export class Suite extends Base implements reporterTypes.Suite { this._entries.push(suite); } + _addAllHook(hook: TestCase) { + hook.parent = this; + this.hooks.push(hook); + } + allTests(): TestCase[] { const result: TestCase[] = []; const visit = (suite: Suite) => { @@ -101,7 +107,7 @@ export class Suite extends Base implements reporterTypes.Suite { suite.location = this.location; suite._requireFile = this._requireFile; suite._use = this._use.slice(); - suite._hooks = this._hooks.slice(); + suite._eachHooks = this._eachHooks.slice(); suite._timeout = this._timeout; suite._annotations = this._annotations.slice(); suite._modifiers = this._modifiers.slice(); @@ -124,19 +130,23 @@ export class TestCase extends Base implements reporterTypes.TestCase { expectedStatus: reporterTypes.TestStatus = 'passed'; timeout = 0; - annotations: Annotation[] = []; + annotations: Annotations = []; retries = 0; repeatEachIndex = 0; + _type: TestCaseType; + _ordinalInFile: number; _testType: TestTypeImpl; _id = ''; _workerHash = ''; _pool: FixturePool | undefined; _projectIndex = 0; - constructor(title: string, fn: Function, testType: TestTypeImpl, location: Location) { + constructor(type: TestCaseType, title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl, location: Location) { super(title); + this._type = type; this.fn = fn; + this._ordinalInFile = ordinalInFile; this._testType = testType; this.location = location; } @@ -164,7 +174,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { } _clone(): TestCase { - const test = new TestCase(this.title, this.fn, this._testType, this.location); + const test = new TestCase(this._type, this.title, this.fn, this._ordinalInFile, this._testType, this.location); test._only = this._only; test._requireFile = this._requireFile; test.expectedStatus = this.expectedStatus; diff --git a/packages/playwright-test/src/testInfo.ts b/packages/playwright-test/src/testInfo.ts index 074f36f631..5a00f5eaf1 100644 --- a/packages/playwright-test/src/testInfo.ts +++ b/packages/playwright-test/src/testInfo.ts @@ -25,15 +25,8 @@ import { WorkerInitParams } from './ipc'; import { Loader } from './loader'; import { ProjectImpl } from './project'; import { TestCase } from './test'; -import { Annotation, TestStepInternal, Location } from './types'; -import { addSuffixToFilePath, getContainedPath, monotonicTime, sanitizeForFilePath, serializeError, trimLongString } from './util'; - -type RunnableDescription = { - type: 'test' | 'beforeAll' | 'afterAll' | 'beforeEach' | 'afterEach' | 'slow' | 'skip' | 'fail' | 'fixme' | 'teardown'; - location?: Location; - // When runnable has a separate timeout, it does not count into the "shared time pool" for the test. - timeout?: number; -}; +import { Annotations, TestStepInternal } from './types'; +import { addSuffixToFilePath, formatLocation, getContainedPath, monotonicTime, sanitizeForFilePath, serializeError, trimLongString } from './util'; export class TestInfoImpl implements TestInfo { private _projectImpl: ProjectImpl; @@ -43,9 +36,6 @@ export class TestInfoImpl implements TestInfo { readonly _startTime: number; readonly _startWallTime: number; private _hasHardError: boolean = false; - private _currentRunnable: RunnableDescription = { type: 'test' }; - // Holds elapsed time of the "time pool" shared between fixtures, each hooks and test itself. - private _elapsedTestTime = 0; // ------------ TestInfo fields ------------ readonly repeatEachIndex: number; @@ -62,7 +52,7 @@ export class TestInfoImpl implements TestInfo { readonly fn: Function; expectedStatus: TestStatus; duration: number = 0; - readonly annotations: Annotation[] = []; + readonly annotations: Annotations = []; readonly attachments: TestInfo['attachments'] = []; status: TestStatus = 'passed'; readonly stdout: TestInfo['stdout'] = []; @@ -126,7 +116,7 @@ export class TestInfoImpl implements TestInfo { const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, '')); const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-'); - const fullTitleWithoutSpec = test.titlePath().slice(1).join(' '); + const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ') + (test._type === 'test' ? '' : '-worker' + this.workerIndex); let testOutputDir = trimLongString(sanitizedRelativePath + '-' + sanitizeForFilePath(fullTitleWithoutSpec)); if (uniqueProjectNamePathSegment) @@ -171,16 +161,6 @@ export class TestInfoImpl implements TestInfo { } } - _setCurrentRunnable(runnable: RunnableDescription) { - if (this._currentRunnable.timeout === undefined) - this._elapsedTestTime = this._timeoutRunner.elapsed(); - this._currentRunnable = runnable; - if (runnable.timeout === undefined) - this._timeoutRunner.updateTimeout(this.timeout, this._elapsedTestTime); - else - this._timeoutRunner.updateTimeout(runnable.timeout, 0); - } - async _runWithTimeout(cb: () => Promise): Promise { try { await this._timeoutRunner.run(cb); @@ -190,15 +170,13 @@ export class TestInfoImpl implements TestInfo { // Do not overwrite existing failure upon hook/teardown timeout. if (this.status === 'passed') { this.status = 'timedOut'; - const title = titleForRunnable(this._currentRunnable); - const suffix = title ? ` in ${title}` : ''; - const message = colors.red(`Timeout of ${this._currentRunnable.timeout ?? this.timeout}ms exceeded${suffix}.`); - const location = this._currentRunnable.location; - this.errors.push({ - message, - // Include location for hooks and modifiers to distinguish between them. - stack: location ? message + `\n at ${location.file}:${location.line}:${location.column}` : undefined, - }); + if (this._test._type === 'test') { + this.errors.push({ message: colors.red(`Timeout of ${this.timeout}ms exceeded.`) }); + } else { + // Include location for the hook to distinguish between multiple hooks. + const message = colors.red(`Timeout of ${this.timeout}ms exceeded in ${this._test._type} hook.`); + this.errors.push({ message: message, stack: message + `\n at ${formatLocation(this._test.location)}.` }); + } } } this.duration = monotonicTime() - this._startTime; @@ -295,38 +273,12 @@ export class TestInfoImpl implements TestInfo { } setTimeout(timeout: number) { - if (this._currentRunnable.timeout !== undefined) { - if (!this._currentRunnable.timeout) - return; // Zero timeout means some debug mode - do not set a timeout. - this._currentRunnable.timeout = timeout; - this._timeoutRunner.updateTimeout(timeout); - } else { - if (!this.timeout) - return; // Zero timeout means some debug mode - do not set a timeout. - this.timeout = timeout; - this._timeoutRunner.updateTimeout(timeout); - } + if (!this.timeout) + return; // Zero timeout means some debug mode - do not set a timeout. + this.timeout = timeout; + this._timeoutRunner.updateTimeout(timeout); } } class SkipError extends Error { } - -function titleForRunnable(runnable: RunnableDescription): string { - switch (runnable.type) { - case 'test': - return ''; - case 'beforeAll': - case 'beforeEach': - case 'afterAll': - case 'afterEach': - return runnable.type + ' hook'; - case 'teardown': - return 'fixtures teardown'; - case 'skip': - case 'slow': - case 'fixme': - case 'fail': - return runnable.type + ' modifier'; - } -} diff --git a/packages/playwright-test/src/testType.ts b/packages/playwright-test/src/testType.ts index 5be9a5e8b6..720decb19f 100644 --- a/packages/playwright-test/src/testType.ts +++ b/packages/playwright-test/src/testType.ts @@ -81,7 +81,7 @@ export class TestTypeImpl { private _createTest(type: 'default' | 'only' | 'skip' | 'fixme', location: Location, title: string, fn: Function) { throwIfRunningInsideJest(); const suite = this._ensureCurrentSuite(location, 'test()'); - const test = new TestCase(title, fn, this, location); + const test = new TestCase('test', title, fn, nextOrdinalInFile(suite._requireFile), this, location); test._requireFile = suite._requireFile; suite._addTest(test); @@ -130,7 +130,15 @@ export class TestTypeImpl { private _hook(name: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll', location: Location, fn: Function) { const suite = this._ensureCurrentSuite(location, `test.${name}()`); - suite._hooks.push({ type: name, fn, location }); + if (name === 'beforeAll' || name === 'afterAll') { + const sameTypeCount = suite.hooks.filter(hook => hook._type === name).length; + const suffix = sameTypeCount ? String(sameTypeCount) : ''; + const hook = new TestCase(name, name + suffix, fn, nextOrdinalInFile(suite._requireFile), this, location); + hook._requireFile = suite._requireFile; + suite._addAllHook(hook); + } else { + suite._eachHooks.push({ type: name, fn, location }); + } } private _configure(location: Location, options: { mode?: 'parallel' | 'serial' }) { @@ -243,4 +251,11 @@ function throwIfRunningInsideJest() { } } +const countByFile = new Map(); +function nextOrdinalInFile(file: string) { + const ordinalInFile = countByFile.get(file) || 0; + countByFile.set(file, ordinalInFile + 1); + return ordinalInFile; +} + export const rootTestType = new TestTypeImpl([]); diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index 0e1838fa5a..bcb8a6f91a 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -23,7 +23,7 @@ export type FixturesWithLocation = { fixtures: Fixtures; location: Location; }; -export type Annotation = { type: string, description?: string }; +export type Annotations = { type: string, description?: string }[]; export interface TestStepInternal { complete(error?: Error | TestError): void; @@ -33,3 +33,5 @@ export interface TestStepInternal { forceNoParent: boolean; location?: Location; } + +export type TestCaseType = 'beforeAll' | 'afterAll' | 'test'; diff --git a/packages/playwright-test/src/worker.ts b/packages/playwright-test/src/worker.ts index 7928e0966f..4e4e8e00a6 100644 --- a/packages/playwright-test/src/worker.ts +++ b/packages/playwright-test/src/worker.ts @@ -84,7 +84,7 @@ process.on('message', async message => { } if (message.method === 'run') { const runPayload = message.params as RunPayload; - await workerRunner!.runTestGroup(runPayload); + await workerRunner!.run(runPayload); } }); diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index b4c06d4259..19b39884c1 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -18,15 +18,15 @@ import rimraf from 'rimraf'; import util from 'util'; import colors from 'colors/safe'; import { EventEmitter } from 'events'; -import { serializeError } from './util'; -import { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload, TeardownErrorsPayload } from './ipc'; +import { serializeError, formatLocation } from './util'; +import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload, TeardownErrorsPayload } from './ipc'; import { setCurrentTestInfo } from './globals'; import { Loader } from './loader'; -import { Suite, TestCase } from './test'; -import { Annotation, TestError, TestStepInternal } from './types'; +import { Modifier, Suite, TestCase } from './test'; +import { Annotations, TestError, TestInfo, TestStepInternal, WorkerInfo } from './types'; import { ProjectImpl } from './project'; import { FixtureRunner } from './fixtures'; -import { ManualPromise, raceAgainstTimeout } from 'playwright-core/lib/utils/async'; +import { raceAgainstTimeout } from 'playwright-core/lib/utils/async'; import { TestInfoImpl } from './testInfo'; const removeFolderAsync = util.promisify(rimraf); @@ -35,25 +35,15 @@ export class WorkerRunner extends EventEmitter { private _params: WorkerInitParams; private _loader!: Loader; private _project!: ProjectImpl; + private _workerInfo!: WorkerInfo; private _fixtureRunner: FixtureRunner; - // Accumulated fatal errors that cannot be attributed to a test. + private _failedTest: TestInfoImpl | undefined; private _fatalErrors: TestError[] = []; - // Whether we should skip running remaining tests in the group because - // of a setup error, usually beforeAll hook. - private _skipRemainingTests = false; - // The stage of the full cleanup. Once "finished", we can safely stop running anything. - private _didRunFullCleanup = false; - // Whether the worker was requested to stop. + private _entries = new Map(); private _isStopped = false; - // This promise resolves once the single "run test group" call finishes. - private _runFinished = new ManualPromise(); + private _runFinished = Promise.resolve(); _currentTest: TestInfoImpl | null = null; - // Dynamic annotations originated by modifiers with a callback, e.g. `test.skip(() => true)`. - private _extraSuiteAnnotations = new Map(); - // Suites that had their beforeAll hooks, but not afterAll hooks executed. - // These suites still need afterAll hooks to be executed for the proper cleanup. - private _activeSuites = new Set(); constructor(params: WorkerInitParams) { super(); @@ -110,7 +100,7 @@ export class WorkerRunner extends EventEmitter { const isExpectError = (error instanceof Error) && !!(error as any).matcherResult; const isCurrentTestExpectedToFail = this._currentTest?.expectedStatus === 'failed'; const shouldConsiderAsTestError = isExpectError || !isCurrentTestExpectedToFail; - if (this._currentTest && shouldConsiderAsTestError) { + if (this._currentTest && this._currentTest._test._type === 'test' && shouldConsiderAsTestError) { this._currentTest._failWithError(serializeError(error), true /* isHardError */); } else { // No current test - fatal error. @@ -126,43 +116,102 @@ export class WorkerRunner extends EventEmitter { this._loader = await Loader.deserialize(this._params.loader); this._project = this._loader.projects()[this._params.projectIndex]; + + this._workerInfo = { + workerIndex: this._params.workerIndex, + parallelIndex: this._params.parallelIndex, + project: this._project.config, + config: this._loader.fullConfig(), + }; } - async runTestGroup(runPayload: RunPayload) { - this._runFinished = new ManualPromise(); + async run(runPayload: RunPayload) { + let runFinishedCallback = () => {}; + this._runFinished = new Promise(f => runFinishedCallback = f); try { - const entries = new Map(runPayload.entries.map(e => [ e.testId, e ])); + this._entries = new Map(runPayload.entries.map(e => [ e.testId, e ])); await this._loadIfNeeded(); const fileSuite = await this._loader.loadTestFile(runPayload.file, 'worker'); const suite = this._project.cloneFileSuite(fileSuite, this._params.repeatEachIndex, test => { - if (!entries.has(test._id)) + if (!this._entries.has(test._id)) return false; return true; }); if (suite) { - this._extraSuiteAnnotations = new Map(); - this._activeSuites = new Set(); - this._didRunFullCleanup = false; - const tests = suite.allTests().filter(test => entries.has(test._id)); - for (let i = 0; i < tests.length; i++) - await this._runTest(tests[i], entries.get(tests[i]._id)!.retry, tests[i + 1]); + const firstPool = suite.allTests()[0]._pool!; + this._fixtureRunner.setPool(firstPool); + await this._runSuite(suite, []); } + if (this._failedTest) + await this._teardownScopes(); } catch (e) { // In theory, we should run above code without any errors. // However, in the case we screwed up, or loadTestFile failed in the worker // but not in the runner, let's do a fatal error. this.unhandledError(e); } finally { + if (this._failedTest) { + // Now that we did run all hooks and teared down scopes, we can + // report the failure, possibly with any error details revealed by teardown. + this.emit('testEnd', buildTestEndPayload(this._failedTest)); + } this._reportDone(); - this._runFinished.resolve(); + runFinishedCallback(); } } - private async _runTest(test: TestCase, retry: number, nextTest: TestCase | undefined) { - // Do not run tests after full cleanup, because we are entirely done. - if (this._isStopped && this._didRunFullCleanup) + private async _runSuite(suite: Suite, annotations: Annotations) { + // When stopped, do not run a suite. But if we have started running the suite with hooks, + // always finish the hooks. + if (this._isStopped) return; + annotations = annotations.concat(suite._annotations); + const allSkipped = suite.allTests().every(test => { + const runEntry = this._entries.get(test._id); + return !runEntry || test.expectedStatus === 'skipped'; + }); + if (allSkipped) { + // This avoids running beforeAll/afterAll hooks. + annotations.push({ type: 'skip' }); + } + + for (const beforeAllModifier of suite._modifiers) { + if (!this._fixtureRunner.dependsOnWorkerFixturesOnly(beforeAllModifier.fn, beforeAllModifier.location)) + continue; + // TODO: separate timeout for beforeAll modifiers? + const result = await raceAgainstTimeout(() => this._fixtureRunner.resolveParametersAndRunFunction(beforeAllModifier.fn, this._workerInfo, undefined), this._project.config.timeout); + if (result.timedOut) { + this._fatalErrors.push(serializeError(new Error(`Timeout of ${this._project.config.timeout}ms exceeded while running ${beforeAllModifier.type} modifier\n at ${formatLocation(beforeAllModifier.location)}`))); + this.stop(); + } else if (!!result.result) { + annotations.push({ type: beforeAllModifier.type, description: beforeAllModifier.description }); + } + } + + for (const hook of suite.hooks) { + if (hook._type !== 'beforeAll') + continue; + const firstTest = suite.allTests()[0]; + await this._runTestOrAllHook(hook, annotations, this._entries.get(firstTest._id)?.retry || 0); + } + for (const entry of suite._entries) { + if (entry instanceof Suite) { + await this._runSuite(entry, annotations); + } else { + const runEntry = this._entries.get(entry._id); + if (runEntry && !this._isStopped) + await this._runTestOrAllHook(entry, annotations, runEntry.retry); + } + } + for (const hook of suite.hooks) { + if (hook._type !== 'afterAll') + continue; + await this._runTestOrAllHook(hook, annotations, 0); + } + } + + private async _runTestOrAllHook(test: TestCase, annotations: Annotations, retry: number) { let lastStepId = 0; const testInfo = new TestInfoImpl(this._loader, this._params, test, retry, data => { const stepId = `${data.category}@${data.title}@${++lastStepId}`; @@ -198,7 +247,16 @@ export class WorkerRunner extends EventEmitter { return step; }); - const processAnnotation = (annotation: Annotation) => { + // Inherit test.setTimeout() from parent suites. + for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) { + if (suite._timeout !== undefined) { + testInfo.setTimeout(suite._timeout); + break; + } + } + + // Process annotations defined on parent suites. + for (const annotation of annotations) { testInfo.annotations.push(annotation); switch (annotation.type) { case 'fixme': @@ -213,35 +271,11 @@ export class WorkerRunner extends EventEmitter { testInfo.setTimeout(testInfo.timeout * 3); break; } - }; - - if (!this._isStopped) { - // Update the fixture pool - it may differ between tests, but only in test-scoped fixtures. - this._fixtureRunner.setPool(test._pool!); - } - - const suites = getSuites(test); - const reversedSuites = suites.slice().reverse(); - - // Inherit test.setTimeout() from parent suites, deepest has the priority. - for (const suite of reversedSuites) { - if (suite._timeout !== undefined) { - testInfo.setTimeout(suite._timeout); - break; - } - } - - // Process existing annotations defined on parent suites. - for (const suite of suites) { - for (const annotation of suite._annotations) - processAnnotation(annotation); - const extraAnnotations = this._extraSuiteAnnotations.get(suite) || []; - for (const annotation of extraAnnotations) - processAnnotation(annotation); } this._currentTest = testInfo; setCurrentTestInfo(testInfo); + this.emit('testBegin', buildTestBeginPayload(testInfo)); if (testInfo.expectedStatus === 'skipped') { @@ -250,137 +284,32 @@ export class WorkerRunner extends EventEmitter { return; } - // Assume beforeAll failed until we actually finish it successfully. - let didFailBeforeAll = true; - let shouldRunAfterEachHooks = false; + // Update the fixture pool - it may differ between tests, but only in test-scoped fixtures. + this._fixtureRunner.setPool(test._pool!); - await testInfo._runWithTimeout(async () => { - if (this._isStopped) { - // Getting here means that worker is requested to stop, but was not able to - // run full cleanup yet. Skip the test, but run the cleanup. - testInfo.status = 'skipped'; - didFailBeforeAll = false; - return; - } - - const beforeHooksStep = testInfo._addStep({ - category: 'hook', - title: 'Before Hooks', - canHaveChildren: true, - forceNoParent: true - }); - - // Note: wrap all preparation steps together, because failure/skip in any of them - // prevents further setup and/or test from running. - const maybeError = await testInfo._runFn(async () => { - // Run "beforeAll" modifiers on parent suites, unless already run during previous tests. - for (const suite of suites) { - if (this._extraSuiteAnnotations.has(suite)) - continue; - const extraAnnotations: Annotation[] = []; - this._extraSuiteAnnotations.set(suite, extraAnnotations); - await this._runModifiersForSuite(suite, testInfo, 'worker', extraAnnotations); - } - - // Run "beforeAll" hooks, unless already run during previous tests. - for (const suite of suites) - await this._runBeforeAllHooksForSuite(suite, testInfo); - // Running "beforeAll" succeeded! - didFailBeforeAll = false; - - // Run "beforeEach" modifiers. - for (const suite of suites) - await this._runModifiersForSuite(suite, testInfo, 'test'); - - // Run "beforeEach" hooks. Once started with "beforeEach", we must run all "afterEach" hooks as well. - shouldRunAfterEachHooks = true; - await this._runEachHooksForSuites(suites, 'beforeEach', testInfo); - - // Setup fixtures required by the test. - testInfo._setCurrentRunnable({ type: 'test' }); - const params = await this._fixtureRunner.resolveParametersForFunction(test.fn, testInfo); - beforeHooksStep.complete(); // Report fixture hooks step as completed. - - // Now run the test itself. - const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]"). - await fn(params, testInfo); - }, 'allowSkips'); - - beforeHooksStep.complete(maybeError); // Second complete is a no-op. - }); - - if (didFailBeforeAll) { - // This will inform dispatcher that we should not run more tests from this group - // because we had a beforeAll error. - // This behavior avoids getting the same common error for each test. - this._skipRemainingTests = true; - } - - const afterHooksStep = testInfo._addStep({ - category: 'hook', - title: 'After Hooks', - canHaveChildren: true, - forceNoParent: true - }); - let firstAfterHooksError: TestError | undefined; + await testInfo._runWithTimeout(() => this._runTestWithBeforeHooks(test, testInfo)); if (testInfo.status === 'timedOut') { // A timed-out test gets a full additional timeout to run after hooks. - testInfo._timeoutRunner.updateTimeout(testInfo.timeout, 0); + testInfo._timeoutRunner.resetTimeout(testInfo.timeout); } - await testInfo._runWithTimeout(async () => { - // Note: do not wrap all teardown steps together, because failure in any of them - // does not prevent further teardown steps from running. + await testInfo._runWithTimeout(() => this._runAfterHooks(test, testInfo)); - // Run "afterEach" hooks, unless we failed at beforeAll stage. - if (shouldRunAfterEachHooks) { - const afterEachError = await testInfo._runFn(() => this._runEachHooksForSuites(reversedSuites, 'afterEach', testInfo)); - firstAfterHooksError = firstAfterHooksError || afterEachError; - } - - // Run "afterAll" hooks for suites that are not shared with the next test. - const nextSuites = new Set(getSuites(nextTest)); - for (const suite of reversedSuites) { - if (!nextSuites.has(suite)) { - const afterAllError = await this._runAfterAllHooksForSuite(suite, testInfo); - firstAfterHooksError = firstAfterHooksError || afterAllError; - } - } - - // Teardown test-scoped fixtures. - testInfo._setCurrentRunnable({ type: 'teardown' }); - const testScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test')); - firstAfterHooksError = firstAfterHooksError || testScopeError; - }); - - const isFailure = testInfo.status !== 'skipped' && testInfo.status !== testInfo.expectedStatus; - if (isFailure) - this._isStopped = true; - - if (this._isStopped) { - // Run all remaining "afterAll" hooks and teardown all fixtures when worker is shutting down. - // Mark as "cleaned up" early to avoid running cleanup twice. - this._didRunFullCleanup = true; - - // Give it more time for the full cleanup. - testInfo._timeoutRunner.updateTimeout(this._project.config.timeout, 0); - await testInfo._runWithTimeout(async () => { - for (const suite of reversedSuites) { - const afterAllError = await this._runAfterAllHooksForSuite(suite, testInfo); - firstAfterHooksError = firstAfterHooksError || afterAllError; - } - testInfo._setCurrentRunnable({ type: 'teardown', timeout: this._project.config.timeout }); - const testScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test')); - firstAfterHooksError = firstAfterHooksError || testScopeError; - const workerScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('worker')); - firstAfterHooksError = firstAfterHooksError || workerScopeError; - }); - } - - afterHooksStep.complete(firstAfterHooksError); this._currentTest = null; setCurrentTestInfo(null); - this.emit('testEnd', buildTestEndPayload(testInfo)); + + const isFailure = testInfo.status !== 'skipped' && testInfo.status !== testInfo.expectedStatus; + if (isFailure) { + // Delay reporting testEnd result until after teardownScopes is done. + this._failedTest = testInfo; + if (test._type !== 'test') { + // beforeAll/afterAll hook failure skips any remaining tests in the worker. + this._fatalErrors.push(...testInfo.errors); + } + this.stop(); + } else { + this.emit('testEnd', buildTestEndPayload(testInfo)); + } const preserveOutput = this._loader.fullConfig().preserveOutput === 'always' || (this._loader.fullConfig().preserveOutput === 'failures-only' && isFailure); @@ -388,63 +317,65 @@ export class WorkerRunner extends EventEmitter { await removeFolderAsync(testInfo.outputDir).catch(e => {}); } - private async _runModifiersForSuite(suite: Suite, testInfo: TestInfoImpl, scope: 'worker' | 'test', extraAnnotations?: Annotation[]) { - for (const modifier of suite._modifiers) { - const actualScope = this._fixtureRunner.dependsOnWorkerFixturesOnly(modifier.fn, modifier.location) ? 'worker' : 'test'; - if (actualScope !== scope) - continue; - testInfo._setCurrentRunnable({ type: modifier.type, location: modifier.location, timeout: scope === 'worker' ? this._project.config.timeout : undefined }); - const result = await this._fixtureRunner.resolveParametersAndRunFunction(modifier.fn, testInfo); - if (result && extraAnnotations) - extraAnnotations.push({ type: modifier.type, description: modifier.description }); - testInfo[modifier.type](!!result, modifier.description); - } - } - - private async _runBeforeAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl) { - if (this._activeSuites.has(suite)) - return; - this._activeSuites.add(suite); - let beforeAllError: Error | undefined; - for (const hook of suite._hooks) { - if (hook.type !== 'beforeAll') - continue; - try { - testInfo._setCurrentRunnable({ type: 'beforeAll', location: hook.location, timeout: this._project.config.timeout }); - await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo); - } catch (e) { - // Always run all the hooks, and capture the first error. - beforeAllError = beforeAllError || e; + private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfoImpl) { + const step = testInfo._addStep({ + category: 'hook', + title: 'Before Hooks', + canHaveChildren: true, + forceNoParent: true + }); + const maybeError = await testInfo._runFn(async () => { + if (test._type === 'test') { + const beforeEachModifiers: Modifier[] = []; + for (let s: Suite | undefined = test.parent; s; s = s.parent) { + const modifiers = s._modifiers.filter(modifier => !this._fixtureRunner.dependsOnWorkerFixturesOnly(modifier.fn, modifier.location)); + beforeEachModifiers.push(...modifiers.reverse()); + } + beforeEachModifiers.reverse(); + for (const modifier of beforeEachModifiers) { + const result = await this._fixtureRunner.resolveParametersAndRunFunction(modifier.fn, this._workerInfo, testInfo); + testInfo[modifier.type](!!result, modifier.description!); + } + await this._runHooks(test.parent!, 'beforeEach', testInfo); } - } - if (beforeAllError) - throw beforeAllError; + + const params = await this._fixtureRunner.resolveParametersForFunction(test.fn, this._workerInfo, testInfo); + step.complete(); // Report fixture hooks step as completed. + const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]"). + await fn(params, testInfo); + }, 'allowSkips'); + step.complete(maybeError); // Second complete is a no-op. } - private async _runAfterAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl) { - if (!this._activeSuites.has(suite)) - return; - this._activeSuites.delete(suite); - let firstError: TestError | undefined; - for (const hook of suite._hooks) { - if (hook.type !== 'afterAll') - continue; - const afterAllError = await testInfo._runFn(async () => { - testInfo._setCurrentRunnable({ type: 'afterAll', location: hook.location, timeout: this._project.config.timeout }); - await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo); - }); - firstError = firstError || afterAllError; - } - return firstError; + private async _runAfterHooks(test: TestCase, testInfo: TestInfoImpl) { + const step = testInfo._addStep({ + category: 'hook', + title: 'After Hooks', + canHaveChildren: true, + forceNoParent: true + }); + + let teardownError1: TestError | undefined; + if (test._type === 'test') + teardownError1 = await testInfo._runFn(() => this._runHooks(test.parent!, 'afterEach', testInfo)); + // Continue teardown even after the failure. + + const teardownError2 = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test')); + step.complete(teardownError1 || teardownError2); } - private async _runEachHooksForSuites(suites: Suite[], type: 'beforeEach' | 'afterEach', testInfo: TestInfoImpl) { - const hooks = suites.map(suite => suite._hooks.filter(hook => hook.type === type)).flat(); + private async _runHooks(suite: Suite, type: 'beforeEach' | 'afterEach', testInfo: TestInfo) { + const all = []; + for (let s: Suite | undefined = suite; s; s = s.parent) { + const funcs = s._eachHooks.filter(e => e.type === type).map(e => e.fn); + all.push(...funcs.reverse()); + } + if (type === 'beforeEach') + all.reverse(); let error: Error | undefined; - for (const hook of hooks) { + for (const hook of all) { try { - testInfo._setCurrentRunnable({ type, location: hook.location }); - await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo); + await this._fixtureRunner.resolveParametersAndRunFunction(hook, this._workerInfo, testInfo); } catch (e) { // Always run all the hooks, and capture the first error. error = error || e; @@ -455,10 +386,10 @@ export class WorkerRunner extends EventEmitter { } private _reportDone() { - const donePayload: DonePayload = { fatalErrors: this._fatalErrors, skipRemaining: this._skipRemainingTests }; + const donePayload: DonePayload = { fatalErrors: this._fatalErrors }; this.emit('done', donePayload); this._fatalErrors = []; - this._skipRemainingTests = false; + this._failedTest = undefined; } } @@ -486,11 +417,3 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload { })) }; } - -function getSuites(test: TestCase | undefined): Suite[] { - const suites: Suite[] = []; - for (let suite: Suite | undefined = test?.parent; suite; suite = suite.parent) - suites.push(suite); - suites.reverse(); // Put root suite first. - return suites; -} diff --git a/tests/playwright-test/fixture-errors.spec.ts b/tests/playwright-test/fixture-errors.spec.ts index a0b457299b..03ba3d5ea4 100644 --- a/tests/playwright-test/fixture-errors.spec.ts +++ b/tests/playwright-test/fixture-errors.spec.ts @@ -453,26 +453,7 @@ test('should not report fixture teardown error twice', async ({ runInlineTest }) expect(result.failed).toBe(1); expect(result.output).toContain('Error: Oh my error'); expect(stripAnsi(result.output)).toContain(`throw new Error('Oh my error')`); - expect(countTimes(stripAnsi(result.output), 'Oh my error')).toBe(2); -}); - -test('should not report fixture teardown timeout twice', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - const test = pwt.test.extend({ - fixture: async ({ }, use) => { - await use(); - await new Promise(() => {}); - }, - }); - test('good', async ({ fixture }) => { - }); - `, - }, { reporter: 'list', timeout: 1000 }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.output).toContain('while shutting down environment'); - expect(countTimes(result.output, 'while shutting down environment')).toBe(1); + expect(countTimes(result.output, 'Oh my error')).toBe(2); }); test('should handle fixture teardown error after test timeout and continue', async ({ runInlineTest }) => { diff --git a/tests/playwright-test/fixtures.spec.ts b/tests/playwright-test/fixtures.spec.ts index 4f93658d16..59c8cd1bb7 100644 --- a/tests/playwright-test/fixtures.spec.ts +++ b/tests/playwright-test/fixtures.spec.ts @@ -316,23 +316,23 @@ test('automatic fixtures should work', async ({ runInlineTest }) => { }); test.beforeEach(async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest === 1 || counterTest === 2).toBe(true); + expect(counterTest === 2 || counterTest === 3).toBe(true); }); test('test 1', async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest).toBe(1); + expect(counterTest).toBe(2); }); test('test 2', async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest).toBe(2); + expect(counterTest).toBe(3); }); test.afterEach(async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest === 1 || counterTest === 2).toBe(true); + expect(counterTest === 2 || counterTest === 3).toBe(true); }); test.afterAll(async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest).toBe(2); + expect(counterTest).toBe(4); }); ` }); diff --git a/tests/playwright-test/hooks.spec.ts b/tests/playwright-test/hooks.spec.ts index 9d0ecda089..de1d31c06c 100644 --- a/tests/playwright-test/hooks.spec.ts +++ b/tests/playwright-test/hooks.spec.ts @@ -63,10 +63,14 @@ test('hooks should work with fixtures', async ({ runInlineTest }) => { '+w', '+t', 'beforeAll-17-42', - 'beforeEach-17-42', - 'test-17-42', - 'afterEach-17-42', - 'afterAll-17-42', + '-t', + '+t', + 'beforeEach-17-43', + 'test-17-43', + 'afterEach-17-43', + '-t', + '+t', + 'afterAll-17-44', '-t', '+t', ]); @@ -92,11 +96,11 @@ test('afterEach failure should not prevent other hooks and fixtures teardown', a const { test } = require('./helper'); test.describe('suite', () => { test.afterEach(async () => { - console.log('afterEach2'); - throw new Error('afterEach2'); + console.log('afterEach1'); }); test.afterEach(async () => { - console.log('afterEach1'); + console.log('afterEach2'); + throw new Error('afterEach2'); }); test('one', async ({foo}) => { console.log('test'); @@ -309,7 +313,7 @@ test('beforeAll hook should get retry index of the first test', async ({ runInli ]); }); -test('afterAll exception should fail the test', async ({ runInlineTest }) => { +test('afterAll exception should fail the run', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.js': ` const { test } = pwt; @@ -321,8 +325,7 @@ test('afterAll exception should fail the test', async ({ runInlineTest }) => { `, }); expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(1); + expect(result.passed).toBe(1); expect(result.output).toContain('From the afterAll'); }); @@ -367,17 +370,13 @@ test('beforeAll failure should prevent the test, but not afterAll', async ({ run test.afterAll(() => { console.log('\\n%%afterAll'); }); - test('failed', () => { - console.log('\\n%%test1'); - }); test('skipped', () => { - console.log('\\n%%test2'); + console.log('\\n%%test'); }); `, }); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); - expect(result.skipped).toBe(1); expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ '%%beforeAll', '%%afterAll', @@ -455,7 +454,7 @@ test('afterAll error should not mask beforeAll', async ({ runInlineTest }) => { expect(result.output).toContain('from beforeAll'); }); -test('beforeAll timeout should be reported and prevent more tests', async ({ runInlineTest }) => { +test('beforeAll timeout should be reported', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.js': ` const { test } = pwt; @@ -466,62 +465,41 @@ test('beforeAll timeout should be reported and prevent more tests', async ({ run test.afterAll(() => { console.log('\\n%%afterAll'); }); - test('failed', () => { - console.log('\\n%%test1'); - }); test('skipped', () => { - console.log('\\n%%test2'); + console.log('\\n%%test'); }); `, }, { timeout: 1000 }); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); - expect(result.skipped).toBe(1); expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ '%%beforeAll', '%%afterAll', ]); expect(result.output).toContain('Timeout of 1000ms exceeded in beforeAll hook.'); - expect(result.output).toContain(`a.test.js:6:12`); - expect(stripAnsi(result.output)).toContain(`> 6 | test.beforeAll(async () => {`); }); -test('afterAll timeout should be reported, run other afterAll hooks, and continue testing', async ({ runInlineTest }, testInfo) => { +test('afterAll timeout should be reported', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'a.test.js': ` const { test } = pwt; - test.describe('suite', () => { - test.afterAll(async () => { - console.log('\\n%%afterAll1'); - await new Promise(f => setTimeout(f, 5000)); - }); - test('runs', () => { - test.setTimeout(2000); - console.log('\\n%%test1'); - }); - }); test.afterAll(async () => { - console.log('\\n%%afterAll2'); + console.log('\\n%%afterAll'); + await new Promise(f => setTimeout(f, 5000)); }); - test('does not run', () => { - console.log('\\n%%test2'); + test('runs', () => { + console.log('\\n%%test'); }); `, }, { timeout: 1000 }); expect(result.exitCode).toBe(1); expect(result.passed).toBe(1); - expect(result.failed).toBe(1); - expect(result.skipped).toBe(0); expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ - '%%test1', - '%%afterAll1', - '%%afterAll2', - '%%test2', - '%%afterAll2', + '%%test', + '%%afterAll', ]); expect(result.output).toContain('Timeout of 1000ms exceeded in afterAll hook.'); - expect(result.output).toContain(`a.test.js:7:14`); - expect(stripAnsi(result.output)).toContain(`> 7 | test.afterAll(async () => {`); + expect(result.output).toContain(`at a.test.js:6:12`); }); test('beforeAll and afterAll timeouts at the same time should be reported', async ({ runInlineTest }) => { @@ -628,127 +606,13 @@ test('should not hang and report results when worker process suddenly exits duri const result = await runInlineTest({ 'a.spec.js': ` const { test } = pwt; - test('failing due to afterall', () => {}); + test('passed', () => {}); test.afterAll(() => { process.exit(0); }); ` }, { reporter: 'line' }); expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(1); + expect(result.passed).toBe(1); expect(result.output).toContain('Worker process exited unexpectedly'); - expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 › failing due to afterall'); -}); - -test('unhandled rejection during beforeAll should be reported and prevent more tests', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - const { test } = pwt; - test.beforeAll(async () => { - console.log('\\n%%beforeAll'); - Promise.resolve().then(() => { - throw new Error('Oh my'); - }); - await new Promise(f => setTimeout(f, 100)); - }); - test.afterAll(() => { - console.log('\\n%%afterAll'); - }); - test('failed', () => { - console.log('\\n%%test1'); - }); - test('skipped', () => { - console.log('\\n%%test2'); - }); - `, - }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.skipped).toBe(1); - expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ - '%%beforeAll', - '%%afterAll', - ]); - expect(result.output).toContain('Error: Oh my'); - expect(stripAnsi(result.output)).toContain(`> 9 | throw new Error('Oh my');`); -}); - -test('beforeAll and afterAll should have a separate timeout', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - const { test } = pwt; - test.beforeAll(async () => { - console.log('\\n%%beforeAll'); - await new Promise(f => setTimeout(f, 300)); - }); - test.beforeAll(async () => { - console.log('\\n%%beforeAll2'); - await new Promise(f => setTimeout(f, 300)); - }); - test('passed', async () => { - console.log('\\n%%test'); - await new Promise(f => setTimeout(f, 300)); - }); - test.afterAll(async () => { - console.log('\\n%%afterAll'); - await new Promise(f => setTimeout(f, 300)); - }); - test.afterAll(async () => { - console.log('\\n%%afterAll2'); - await new Promise(f => setTimeout(f, 300)); - }); - `, - }, { timeout: '500' }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ - '%%beforeAll', - '%%beforeAll2', - '%%test', - '%%afterAll', - '%%afterAll2', - ]); -}); - -test('test.setTimeout should work separately in beforeAll', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - const { test } = pwt; - test.beforeAll(async () => { - console.log('\\n%%beforeAll'); - test.setTimeout(100); - }); - test('passed', async () => { - console.log('\\n%%test'); - await new Promise(f => setTimeout(f, 800)); - }); - `, - }, { timeout: '1000' }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ - '%%beforeAll', - '%%test', - ]); -}); - -test('test.setTimeout should work separately in afterAll', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - const { test } = pwt; - test('passed', async () => { - console.log('\\n%%test'); - }); - test.afterAll(async () => { - console.log('\\n%%afterAll'); - test.setTimeout(1000); - await new Promise(f => setTimeout(f, 800)); - }); - `, - }, { timeout: '100' }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ - '%%test', - '%%afterAll', - ]); + expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 › passed'); + expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:7:12 › afterAll'); }); diff --git a/tests/playwright-test/loader.spec.ts b/tests/playwright-test/loader.spec.ts index 2280c602e7..d596947b88 100644 --- a/tests/playwright-test/loader.spec.ts +++ b/tests/playwright-test/loader.spec.ts @@ -407,52 +407,3 @@ test('should filter stack even without default Error.prepareStackTrace', async ( expect(stackLines.length).toBe(1); }); -test('should work with cross-imports - 1', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'test1.spec.ts': ` - const { test } = pwt; - test('test 1', async ({}) => { - await new Promise(x => setTimeout(x, 500)); - console.log('running TEST-1'); - }); - `, - 'test2.spec.ts': ` - import * as _ from './test1.spec'; - const { test } = pwt; - test('test 2', async ({}) => { - await new Promise(x => setTimeout(x, 500)); - console.log('running TEST-2'); - }); - ` - }, { workers: 2 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(2); - expect(result.failed).toBe(0); - expect(result.output).toContain('TEST-1'); - expect(result.output).toContain('TEST-2'); -}); - -test('should work with cross-imports - 2', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'test1.spec.ts': ` - const { test } = pwt; - import * as _ from './test2.spec'; - test('test 1', async ({}) => { - await new Promise(x => setTimeout(x, 500)); - console.log('running TEST-1'); - }); - `, - 'test2.spec.ts': ` - const { test } = pwt; - test('test 2', async ({}) => { - await new Promise(x => setTimeout(x, 500)); - console.log('running TEST-2'); - }); - ` - }, { workers: 2, reporter: 'list' }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(2); - expect(result.failed).toBe(0); - expect(result.output).toContain('TEST-1'); - expect(result.output).toContain('TEST-2'); -}); diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.ts index 080c4dc92d..a48788ed56 100644 --- a/tests/playwright-test/playwright.artifacts.spec.ts +++ b/tests/playwright-test/playwright.artifacts.spec.ts @@ -46,7 +46,6 @@ const testFiles = { }); test.afterAll(async () => { - await page.setContent('Reset!'); await page.close(); }); @@ -146,6 +145,10 @@ test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => { ' test-failed-1.png', 'artifacts-persistent-passing', ' test-finished-1.png', + 'artifacts-shared-afterAll-worker0', + ' test-finished-1.png', + 'artifacts-shared-beforeAll-worker0', + ' test-finished-1.png', 'artifacts-shared-shared-failing', ' test-failed-1.png', 'artifacts-shared-shared-passing', @@ -211,6 +214,10 @@ test('should work with trace: on', async ({ runInlineTest }, testInfo) => { ' trace.zip', 'artifacts-persistent-passing', ' trace.zip', + 'artifacts-shared-afterAll-worker0', + ' trace.zip', + 'artifacts-shared-beforeAll-worker0', + ' trace.zip', 'artifacts-shared-shared-failing', ' trace.zip', 'artifacts-shared-shared-passing', @@ -270,6 +277,8 @@ test('should work with trace: on-first-retry', async ({ runInlineTest }, testInf ' trace.zip', 'artifacts-persistent-failing-retry1', ' trace.zip', + 'artifacts-shared-beforeAll-worker1-retry1', + ' trace.zip', 'artifacts-shared-shared-failing-retry1', ' trace.zip', 'artifacts-two-contexts-failing-retry1', diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 68a830039b..f1fc09bca3 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -343,11 +343,15 @@ test('should report api steps', async ({ runInlineTest }) => { `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, `%% end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"},{\"title\":\"browserContext.close\",\"category\":\"pw:api\"}]}`, `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, `%% begin {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`, `%% end {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`, `%% begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, `%% end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, - `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"browser.newPage\",\"category\":\"pw:api\"},{\"title\":\"page.setContent\",\"category\":\"pw:api\"}]}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, `%% begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, `%% end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, @@ -357,9 +361,13 @@ test('should report api steps', async ({ runInlineTest }) => { `%% begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, `%% end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, `%% begin {\"title\":\"page.close\",\"category\":\"pw:api\"}`, `%% end {\"title\":\"page.close\",\"category\":\"pw:api\"}`, - `%% end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, ]); }); diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.ts index d85b93c79b..8c2af4deea 100644 --- a/tests/playwright-test/test-modifiers.spec.ts +++ b/tests/playwright-test/test-modifiers.spec.ts @@ -330,38 +330,6 @@ test('modifier timeout should be reported', async ({ runInlineTest }) => { }, { timeout: 2000 }); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); - expect(result.output).toContain('Timeout of 2000ms exceeded in skip modifier.'); + expect(result.output).toContain('Error: Timeout of 2000ms exceeded while running skip modifier'); expect(stripAnsi(result.output)).toContain('6 | test.skip(async () => new Promise(() => {}));'); }); - -test('should not run hooks if modifier throws', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - const { test } = pwt; - test.skip(() => { - console.log('%%modifier'); - throw new Error('Oh my'); - }); - test.beforeAll(() => { - console.log('%%beforeEach'); - }); - test.beforeEach(() => { - console.log('%%beforeEach'); - }); - test.afterEach(() => { - console.log('%%afterEach'); - }); - test.afterAll(() => { - console.log('%%beforeEach'); - }); - test('skipped1', () => { - console.log('%%skipped1'); - }); - `, - }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ - '%%modifier', - ]); -}); diff --git a/tests/playwright-test/test-output-dir.spec.ts b/tests/playwright-test/test-output-dir.spec.ts index 0e2a887b15..0c8f976408 100644 --- a/tests/playwright-test/test-output-dir.spec.ts +++ b/tests/playwright-test/test-output-dir.spec.ts @@ -83,27 +83,53 @@ test('should include repeat token', async ({ runInlineTest }) => { expect(result.passed).toBe(3); }); -test('should be unique for beforeAll hook from different workers', async ({ runInlineTest }, testInfo) => { +test('should be unique for beforeAll and afterAll hooks', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'a.spec.js': ` const { test } = pwt; test.beforeAll(({}, testInfo) => { console.log('\\n%%' + testInfo.outputDir); }); - test('fails', ({}, testInfo) => { - expect(1).toBe(2); + test.beforeAll(({}, testInfo) => { + console.log('\\n%%' + testInfo.outputDir); }); - test('passes', ({}, testInfo) => { + test.afterAll(({}, testInfo) => { + console.log('\\n%%' + testInfo.outputDir); + }); + test.afterAll(({}, testInfo) => { + console.log('\\n%%' + testInfo.outputDir); + }); + test.describe('suite', () => { + test.beforeAll(({}, testInfo) => { + console.log('\\n%%' + testInfo.outputDir); + }); + test.afterAll(({}, testInfo) => { + console.log('\\n%%' + testInfo.outputDir); + }); + test('fails', ({}, testInfo) => { + expect(1).toBe(2); + }); + test('passes', ({}, testInfo) => { + }); }); ` - }, { retries: '1' }); + }); expect(result.exitCode).toBe(1); expect(result.passed).toBe(1); expect(result.failed).toBe(1); expect(result.output.split('\n').filter(x => x.startsWith('%%'))).toEqual([ - `%%${testInfo.outputPath('test-results', 'a-fails')}`, - `%%${testInfo.outputPath('test-results', 'a-fails-retry1')}`, - `%%${testInfo.outputPath('test-results', 'a-passes')}`, + `%%${testInfo.outputPath('test-results', 'a-beforeAll-worker0')}`, + `%%${testInfo.outputPath('test-results', 'a-beforeAll1-worker0')}`, + `%%${testInfo.outputPath('test-results', 'a-suite-beforeAll-worker0')}`, + `%%${testInfo.outputPath('test-results', 'a-suite-afterAll-worker0')}`, + `%%${testInfo.outputPath('test-results', 'a-afterAll-worker0')}`, + `%%${testInfo.outputPath('test-results', 'a-afterAll1-worker0')}`, + `%%${testInfo.outputPath('test-results', 'a-beforeAll-worker1')}`, + `%%${testInfo.outputPath('test-results', 'a-beforeAll1-worker1')}`, + `%%${testInfo.outputPath('test-results', 'a-suite-beforeAll-worker1')}`, + `%%${testInfo.outputPath('test-results', 'a-suite-afterAll-worker1')}`, + `%%${testInfo.outputPath('test-results', 'a-afterAll-worker1')}`, + `%%${testInfo.outputPath('test-results', 'a-afterAll1-worker1')}`, ]); });