From 87548f94c16454e8ef197dd8c4ec605842f423a8 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Mon, 9 Aug 2021 13:26:33 -0700 Subject: [PATCH] feat(test runner): support test fixtures in beforeAll/afterAll (#8082) Each hook gets its own test scope. This is not too useful for object fixtures like `page` (although one can use a page in `beforeAll` to save storage state), but much more useful for option fixtures like `viewport`. --- docs/src/test-api/class-test.md | 8 +- src/test/fixtures.ts | 30 +++---- src/test/project.ts | 17 +++- src/test/test.ts | 20 +++-- src/test/testType.ts | 10 ++- src/test/types.ts | 1 + src/test/workerRunner.ts | 92 +++++++++++--------- tests/playwright-test/fixture-errors.spec.ts | 38 -------- tests/playwright-test/fixtures.spec.ts | 62 +++++++++++-- tests/playwright-test/hooks.spec.ts | 64 ++++++++++---- tests/playwright-test/types-2.spec.ts | 2 + tests/playwright-test/types.spec.ts | 15 ---- types/test.d.ts | 8 +- utils/generate_types/overrides-test.d.ts | 4 +- 14 files changed, 213 insertions(+), 158 deletions(-) diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 6e999bb225..7c8137e637 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -64,9 +64,9 @@ Test function that takes one or two arguments: an object with fixtures and optio Declares an `afterAll` hook that is executed once after all tests. When called in the scope of a test file, runs after all tests in the file. When called inside a [`method: Test.describe`] group, runs after all tests in the group. ### param: Test.afterAll.hookFunction -- `hookFunction` <[function]\([Fixtures], [WorkerInfo]\)> +- `hookFunction` <[function]\([Fixtures], [TestInfo]\)> -Hook function that takes one or two arguments: an object with fixtures and optional [WorkerInfo]. +Hook function that takes one or two arguments: an object with fixtures and optional [TestInfo]. @@ -121,9 +121,9 @@ test('my test', async ({ page }) => { You can use [`method: Test.afterAll`] to teardown any resources set up in `beforeAll`. ### param: Test.beforeAll.hookFunction -- `hookFunction` <[function]\([Fixtures], [WorkerInfo]\)> +- `hookFunction` <[function]\([Fixtures], [TestInfo]\)> -Hook function that takes one or two arguments: an object with fixtures and optional [WorkerInfo]. +Hook function that takes one or two arguments: an object with fixtures and optional [TestInfo]. diff --git a/src/test/fixtures.ts b/src/test/fixtures.ts index f963a993a4..d29b51e9f8 100644 --- a/src/test/fixtures.ts +++ b/src/test/fixtures.ts @@ -47,7 +47,7 @@ class Fixture { this.value = null; } - async setup(info: any) { + async setup(workerInfo: WorkerInfo, testInfo: TestInfo | undefined) { if (typeof this.registration.fn !== 'function') { this._setup = true; this.value = this.registration.fn; @@ -57,7 +57,7 @@ class Fixture { const params: { [key: string]: any } = {}; for (const name of this.registration.deps) { const registration = this.runner.pool!.resolveDependency(this.registration, name)!; - const dep = await this.runner.setupFixtureForRegistration(registration, info); + const dep = await this.runner.setupFixtureForRegistration(registration, workerInfo, testInfo); dep.usages.add(this); params[name] = dep.value; } @@ -74,7 +74,7 @@ class Fixture { this.value = value; setupFenceFulfill(); return await teardownFence; - }, info)).catch((e: any) => { + }, this.registration.scope === 'worker' ? workerInfo : testInfo)).catch((e: any) => { if (!this._setup) setupFenceReject(e); else @@ -187,7 +187,7 @@ export class FixturePool { return hash.digest('hex'); } - validateFunction(fn: Function, prefix: string, allowTestFixtures: boolean, location: Location) { + validateFunction(fn: Function, prefix: string, location: Location) { const visit = (registration: FixtureRegistration) => { for (const name of registration.deps) visit(this.resolveDependency(registration, name)!); @@ -196,8 +196,6 @@ export class FixturePool { const registration = this.registrations.get(name); if (!registration) throw errorWithLocations(`${prefix} has unknown parameter "${name}".`, { location, name: prefix, quoted: false }); - if (!allowTestFixtures && registration.scope === 'test') - throw errorWithLocations(`${prefix} cannot depend on a test fixture "${name}".`, { location, name: prefix, quoted: false }, registration); visit(registration); } } @@ -222,8 +220,10 @@ export class FixtureRunner { this.pool = pool; } - async teardownScope(scope: string) { - for (const [, fixture] of this.instanceForId) { + async teardownScope(scope: FixtureScope) { + // Teardown fixtures in the reverse order. + const fixtures = Array.from(this.instanceForId.values()).reverse(); + for (const fixture of fixtures) { if (fixture.registration.scope === scope) await fixture.teardown(); } @@ -231,12 +231,12 @@ export class FixtureRunner { this.testScopeClean = true; } - async resolveParametersAndRunHookOrTest(fn: Function, scope: FixtureScope, info: WorkerInfo|TestInfo) { + async resolveParametersAndRunHookOrTest(fn: Function, workerInfo: WorkerInfo, testInfo: TestInfo | undefined) { // Install all automatic fixtures. for (const registration of this.pool!.registrations.values()) { - const shouldSkip = scope === 'worker' && registration.scope === 'test'; + const shouldSkip = !testInfo && registration.scope === 'test'; if (registration.auto && !shouldSkip) - await this.setupFixtureForRegistration(registration, info); + await this.setupFixtureForRegistration(registration, workerInfo, testInfo); } // Install used fixtures. @@ -244,14 +244,14 @@ export class FixtureRunner { const params: { [key: string]: any } = {}; for (const name of names) { const registration = this.pool!.registrations.get(name)!; - const fixture = await this.setupFixtureForRegistration(registration, info); + const fixture = await this.setupFixtureForRegistration(registration, workerInfo, testInfo); params[name] = fixture.value; } - return fn(params, info); + return fn(params, testInfo || workerInfo); } - async setupFixtureForRegistration(registration: FixtureRegistration, info: any): Promise { + async setupFixtureForRegistration(registration: FixtureRegistration, workerInfo: WorkerInfo, testInfo: TestInfo | undefined): Promise { if (registration.scope === 'test') this.testScopeClean = false; @@ -261,7 +261,7 @@ export class FixtureRunner { fixture = new Fixture(this, registration); this.instanceForId.set(registration.id, fixture); - await fixture.setup(info); + await fixture.setup(workerInfo, testInfo); return fixture; } diff --git a/src/test/project.ts b/src/test/project.ts index db67fb39e7..94e79b0f6a 100644 --- a/src/test/project.ts +++ b/src/test/project.ts @@ -67,18 +67,27 @@ export class ProjectImpl { } this.testPools.set(test, pool); - pool.validateFunction(test.fn, 'Test', true, test.location); + pool.validateFunction(test.fn, 'Test', test.location); for (let parent = test.parent; parent; parent = parent.parent) { - for (const hook of parent._hooks) - pool.validateFunction(hook.fn, hook.type + ' hook', hook.type === 'beforeEach' || hook.type === 'afterEach', hook.location); + for (const hook of parent._eachHooks) + pool.validateFunction(hook.fn, hook.type + ' hook', hook.location); + for (const hook of parent._allHooks) + pool.validateFunction(hook.fn, hook._type + ' hook', hook.location); for (const modifier of parent._modifiers) - pool.validateFunction(modifier.fn, modifier.type + ' modifier', true, modifier.location); + pool.validateFunction(modifier.fn, modifier.type + ' modifier', modifier.location); } } return this.testPools.get(test)!; } private _cloneEntries(from: Suite, to: Suite, repeatEachIndex: number, filter: (test: TestCase) => boolean): boolean { + for (const hook of from._allHooks) { + const clone = hook._clone(); + clone.projectName = this.config.name; + clone._pool = this.buildPool(hook); + clone._projectIndex = this.index; + to._addAllHook(clone); + } for (const entry of from._entries) { if (entry instanceof Suite) { const suite = entry._clone(); diff --git a/src/test/test.ts b/src/test/test.ts index ca7e8cadd1..c6712cb997 100644 --- a/src/test/test.ts +++ b/src/test/test.ts @@ -50,11 +50,8 @@ export class Suite extends Base implements reporterTypes.Suite { location?: Location; _fixtureOverrides: any = {}; _entries: (Suite | TestCase)[] = []; - _hooks: { - type: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll', - fn: Function, - location: Location, - }[] = []; + _allHooks: TestCase[] = []; + _eachHooks: { type: 'beforeEach' | 'afterEach', fn: Function, location: Location }[] = []; _timeout: number | undefined; _annotations: Annotations = []; _modifiers: Modifier[] = []; @@ -71,6 +68,11 @@ export class Suite extends Base implements reporterTypes.Suite { this._entries.push(suite); } + _addAllHook(hook: TestCase) { + hook.parent = this; + this._allHooks.push(hook); + } + allTests(): TestCase[] { const result: TestCase[] = []; const visit = (suite: Suite) => { @@ -105,7 +107,7 @@ export class Suite extends Base implements reporterTypes.Suite { suite.location = this.location; suite._requireFile = this._requireFile; suite._fixtureOverrides = this._fixtureOverrides; - suite._hooks = this._hooks.slice(); + suite._eachHooks = this._eachHooks.slice(); suite._timeout = this._timeout; suite._annotations = this._annotations.slice(); suite._modifiers = this._modifiers.slice(); @@ -124,6 +126,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { projectName = ''; retries = 0; + _type: 'beforeAll' | 'afterAll' | 'test'; _ordinalInFile: number; _testType: TestTypeImpl; _id = ''; @@ -132,8 +135,9 @@ export class TestCase extends Base implements reporterTypes.TestCase { _repeatEachIndex = 0; _projectIndex = 0; - constructor(title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl, location: Location) { + constructor(type: 'beforeAll' | 'afterAll' | 'test', title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl, location: Location) { super(title); + this._type = type; this.fn = fn; this._ordinalInFile = ordinalInFile; this._testType = testType; @@ -156,7 +160,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { } _clone(): TestCase { - const test = new TestCase(this.title, this.fn, this._ordinalInFile, this._testType, this.location); + const test = new TestCase(this._type, this.title, this.fn, this._ordinalInFile, this._testType, this.location); test._only = this._only; test._requireFile = this._requireFile; test.expectedStatus = this.expectedStatus; diff --git a/src/test/testType.ts b/src/test/testType.ts index 89c451ed37..0b3e0f3974 100644 --- a/src/test/testType.ts +++ b/src/test/testType.ts @@ -65,7 +65,7 @@ export class TestTypeImpl { const ordinalInFile = countByFile.get(suite._requireFile) || 0; countByFile.set(suite._requireFile, ordinalInFile + 1); - const test = new TestCase(title, fn, ordinalInFile, this, location); + const test = new TestCase('test', title, fn, ordinalInFile, this, location); test._requireFile = suite._requireFile; suite._addTest(test); @@ -107,7 +107,13 @@ export class TestTypeImpl { const suite = currentlyLoadingFileSuite(); if (!suite) throw errorWithLocation(location, `${name} hook can only be called in a test file`); - suite._hooks.push({ type: name, fn, location }); + if (name === 'beforeAll' || name === 'afterAll') { + const hook = new TestCase(name, name, fn, 0, this, location); + hook._requireFile = suite._requireFile; + suite._addAllHook(hook); + } else { + suite._eachHooks.push({ type: name, fn, location }); + } } private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, ...modifierArgs: [arg?: any | Function, description?: string]) { diff --git a/src/test/types.ts b/src/test/types.ts index aade551240..3515ce4291 100644 --- a/src/test/types.ts +++ b/src/test/types.ts @@ -29,5 +29,6 @@ export type CompleteStepCallback = (error?: Error | TestError) => void; export interface TestInfoImpl extends TestInfo { _testFinished: Promise; + _type: 'test' | 'beforeAll' | 'afterAll'; _addStep: (category: string, title: string) => CompleteStepCallback; } diff --git a/src/test/workerRunner.ts b/src/test/workerRunner.ts index 9e26ab4362..8ae6182529 100644 --- a/src/test/workerRunner.ts +++ b/src/test/workerRunner.ts @@ -75,7 +75,7 @@ export class WorkerRunner extends EventEmitter { unhandledError(error: Error | any) { if (this._isStopped) return; - if (this._currentTest) { + if (this._currentTest && this._currentTest.testInfo._type === 'test') { if (this._currentTest.testInfo.error) return; this._currentTest.testInfo.status = 'failed'; @@ -153,7 +153,7 @@ export class WorkerRunner extends EventEmitter { if (!this._fixtureRunner.dependsOnWorkerFixturesOnly(beforeAllModifier.fn, beforeAllModifier.location)) continue; // TODO: separate timeout for beforeAll modifiers? - const result = await raceAgainstDeadline(this._fixtureRunner.resolveParametersAndRunHookOrTest(beforeAllModifier.fn, 'worker', this._workerInfo), this._deadline()); + const result = await raceAgainstDeadline(this._fixtureRunner.resolveParametersAndRunHookOrTest(beforeAllModifier.fn, this._workerInfo, undefined), this._deadline()); if (result.timedOut) { this._fatalError = serializeError(new Error(`Timeout of ${this._project.config.timeout}ms exceeded while running ${beforeAllModifier.type} modifier`)); this._reportDoneAndStop(); @@ -162,46 +162,37 @@ export class WorkerRunner extends EventEmitter { annotations.push({ type: beforeAllModifier.type, description: beforeAllModifier.description }); } - const skipHooks = annotations.some(a => a.type === 'fixme' || a.type === 'skip'); - for (const hook of suite._hooks) { - if (hook.type !== 'beforeAll' || skipHooks) + for (const hook of suite._allHooks) { + if (hook._type !== 'beforeAll') continue; if (this._isStopped) return; - // TODO: separate timeout for beforeAll? - const result = await raceAgainstDeadline(this._fixtureRunner.resolveParametersAndRunHookOrTest(hook.fn, 'worker', this._workerInfo), this._deadline()); - if (result.timedOut) { - this._fatalError = serializeError(new Error(`Timeout of ${this._project.config.timeout}ms exceeded while running beforeAll hook`)); - this._reportDoneAndStop(); - } + const firstTest = suite.allTests()[0]; + await this._runTestOrAllHook(hook, annotations, this._entries.get(firstTest._id)?.retry || 0); } for (const entry of suite._entries) { - if (entry instanceof Suite) + if (entry instanceof Suite) { await this._runSuite(entry, annotations); - else - await this._runTest(entry, annotations); + } else { + const runEntry = this._entries.get(entry._id); + if (runEntry) + await this._runTestOrAllHook(entry, annotations, runEntry.retry); + } } - for (const hook of suite._hooks) { - if (hook.type !== 'afterAll' || skipHooks) + for (const hook of suite._allHooks) { + if (hook._type !== 'afterAll') continue; if (this._isStopped) return; - // TODO: separate timeout for afterAll? - const result = await raceAgainstDeadline(this._fixtureRunner.resolveParametersAndRunHookOrTest(hook.fn, 'worker', this._workerInfo), this._deadline()); - if (result.timedOut) { - this._fatalError = serializeError(new Error(`Timeout of ${this._project.config.timeout}ms exceeded while running afterAll hook`)); - this._reportDoneAndStop(); - } + await this._runTestOrAllHook(hook, annotations, 0); } } - private async _runTest(test: TestCase, annotations: Annotations) { + private async _runTestOrAllHook(test: TestCase, annotations: Annotations, retry: number) { if (this._isStopped) return; - const entry = this._entries.get(test._id); - if (!entry) - return; + const reportEvents = test._type === 'test'; const startTime = monotonicTime(); const startWallTime = Date.now(); let deadlineRunner: DeadlineRunner | undefined; @@ -213,8 +204,8 @@ export class WorkerRunner extends EventEmitter { let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(test.title); if (this._uniqueProjectNamePathSegment) testOutputDir += '-' + this._uniqueProjectNamePathSegment; - if (entry.retry) - testOutputDir += '-retry' + entry.retry; + if (retry) + testOutputDir += '-retry' + retry; if (this._params.repeatEachIndex) testOutputDir += '-repeat' + this._params.repeatEachIndex; return path.join(this._project.config.outputDir, testOutputDir); @@ -223,14 +214,16 @@ export class WorkerRunner extends EventEmitter { let testFinishedCallback = () => {}; let lastStepId = 0; const testInfo: TestInfoImpl = { - ...this._workerInfo, + workerIndex: this._params.workerIndex, + project: this._project.config, + config: this._loader.fullConfig(), title: test.title, file: test.location.file, line: test.location.line, column: test.location.column, fn: test.fn, repeatEachIndex: this._params.repeatEachIndex, - retry: entry.retry, + retry, expectedStatus: test.expectedStatus, annotations: [], attachments: [], @@ -277,7 +270,8 @@ export class WorkerRunner extends EventEmitter { title, wallTime: Date.now() }; - this.emit('stepBegin', payload); + if (reportEvents) + this.emit('stepBegin', payload); return (error?: Error | TestError) => { if (error instanceof Error) error = serializeError(error); @@ -287,9 +281,11 @@ export class WorkerRunner extends EventEmitter { wallTime: Date.now(), error }; - this.emit('stepEnd', payload); + if (reportEvents) + this.emit('stepEnd', payload); }; }, + _type: test._type, }; // Inherit test.setTimeout() from parent suites. @@ -323,11 +319,13 @@ export class WorkerRunner extends EventEmitter { return testInfo.timeout ? startTime + testInfo.timeout : undefined; }; - this.emit('testBegin', buildTestBeginPayload(testId, testInfo, startWallTime)); + if (reportEvents) + this.emit('testBegin', buildTestBeginPayload(testId, testInfo, startWallTime)); if (testInfo.expectedStatus === 'skipped') { testInfo.status = 'skipped'; - this.emit('testEnd', buildTestEndPayload(testId, testInfo)); + if (reportEvents) + this.emit('testEnd', buildTestEndPayload(testId, testInfo)); return; } @@ -360,7 +358,8 @@ export class WorkerRunner extends EventEmitter { } testInfo.duration = monotonicTime() - startTime; - this.emit('testEnd', buildTestEndPayload(testId, testInfo)); + if (reportEvents) + this.emit('testEnd', buildTestEndPayload(testId, testInfo)); const isFailure = testInfo.status === 'timedOut' || (testInfo.status === 'failed' && testInfo.expectedStatus !== 'failed'); const preserveOutput = this._loader.fullConfig().preserveOutput === 'always' || @@ -373,7 +372,10 @@ export class WorkerRunner extends EventEmitter { if (this._isStopped) return; if (testInfo.status !== 'passed' && testInfo.status !== 'skipped') { - this._failedTestId = testId; + if (test._type === 'test') + this._failedTestId = testId; + else + this._fatalError = testInfo.error; this._reportDoneAndStop(); } } @@ -383,7 +385,7 @@ export class WorkerRunner extends EventEmitter { setCurrentTestInfo(currentTest ? currentTest.testInfo : null); } - private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfoImpl) { + private async _runBeforeHooks(test: TestCase, testInfo: TestInfoImpl) { let completeStep: CompleteStepCallback | undefined; try { const beforeEachModifiers: Modifier[] = []; @@ -395,7 +397,7 @@ export class WorkerRunner extends EventEmitter { for (const modifier of beforeEachModifiers) { if (this._isStopped) return; - const result = await this._fixtureRunner.resolveParametersAndRunHookOrTest(modifier.fn, 'test', testInfo); + const result = await this._fixtureRunner.resolveParametersAndRunHookOrTest(modifier.fn, this._workerInfo, testInfo); testInfo[modifier.type](!!result, modifier.description!); } completeStep = testInfo._addStep('hook', 'Before Hooks'); @@ -411,13 +413,18 @@ export class WorkerRunner extends EventEmitter { // Continue running afterEach hooks even after the failure. } completeStep?.(testInfo.error); + } + + private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfoImpl) { + if (test._type === 'test') + await this._runBeforeHooks(test, testInfo); // Do not run the test when beforeEach hook fails. if (this._isStopped || testInfo.status === 'failed' || testInfo.status === 'skipped') return; try { - await this._fixtureRunner.resolveParametersAndRunHookOrTest(test.fn, 'test', testInfo); + await this._fixtureRunner.resolveParametersAndRunHookOrTest(test.fn, this._workerInfo, testInfo); } catch (error) { if (error instanceof SkipError) { if (testInfo.status === 'passed') @@ -439,7 +446,8 @@ export class WorkerRunner extends EventEmitter { let teardownError: TestError | undefined; try { completeStep = testInfo._addStep('hook', 'After Hooks'); - await this._runHooks(test.parent!, 'afterEach', testInfo); + if (test._type === 'test') + await this._runHooks(test.parent!, 'afterEach', testInfo); } catch (error) { if (!(error instanceof SkipError)) { if (testInfo.status === 'passed') @@ -469,7 +477,7 @@ export class WorkerRunner extends EventEmitter { return; const all = []; for (let s: Suite | undefined = suite; s; s = s.parent) { - const funcs = s._hooks.filter(e => e.type === type).map(e => e.fn); + const funcs = s._eachHooks.filter(e => e.type === type).map(e => e.fn); all.push(...funcs.reverse()); } if (type === 'beforeEach') @@ -477,7 +485,7 @@ export class WorkerRunner extends EventEmitter { let error: Error | undefined; for (const hook of all) { try { - await this._fixtureRunner.resolveParametersAndRunHookOrTest(hook, 'test', testInfo); + await this._fixtureRunner.resolveParametersAndRunHookOrTest(hook, this._workerInfo, testInfo); } catch (e) { // Always run all the hooks, and capture the first error. error = error || e; diff --git a/tests/playwright-test/fixture-errors.spec.ts b/tests/playwright-test/fixture-errors.spec.ts index 7e01ee2b52..b5ba35548e 100644 --- a/tests/playwright-test/fixture-errors.spec.ts +++ b/tests/playwright-test/fixture-errors.spec.ts @@ -179,44 +179,6 @@ test('should throw when worker fixture depends on a test fixture', async ({ runI expect(result.exitCode).toBe(1); }); -test('should throw when beforeAll hook depends on a test fixture', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'f.spec.ts': ` - const test = pwt.test.extend({ - foo: [async ({}, runTest) => { - await runTest(); - }, { scope: 'test' }], - }); - - test.beforeAll(async ({ foo }) => {}); - test('works', async ({ foo }) => {}); - `, - }); - expect(result.output).toContain('beforeAll hook cannot depend on a test fixture "foo".'); - expect(result.output).toContain(`f.spec.ts:11:12`); - expect(result.output).toContain(`f.spec.ts:5:29`); - expect(result.exitCode).toBe(1); -}); - -test('should throw when afterAll hook depends on a test fixture', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'f.spec.ts': ` - const test = pwt.test.extend({ - foo: [async ({}, runTest) => { - await runTest(); - }, { scope: 'test' }], - }); - - test.afterAll(async ({ foo }) => {}); - test('works', async ({ foo }) => {}); - `, - }); - expect(result.output).toContain('afterAll hook cannot depend on a test fixture "foo".'); - expect(result.output).toContain(`f.spec.ts:11:12`); - expect(result.output).toContain(`f.spec.ts:5:29`); - expect(result.exitCode).toBe(1); -}); - test('should define the same fixture in two files', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.spec.ts': ` diff --git a/tests/playwright-test/fixtures.spec.ts b/tests/playwright-test/fixtures.spec.ts index 130a0554e5..5248d00f68 100644 --- a/tests/playwright-test/fixtures.spec.ts +++ b/tests/playwright-test/fixtures.spec.ts @@ -312,27 +312,27 @@ test('automatic fixtures should work', async ({ runInlineTest }) => { }); test.beforeAll(async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest).toBe(0); + expect(counterTest).toBe(1); }); test.beforeEach(async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest === 1 || counterTest === 2).toBe(true); + expect(counterTest === 2 || counterTest === 3).toBe(true); }); test('test 1', async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest).toBe(1); + expect(counterTest).toBe(2); }); test('test 2', async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest).toBe(2); + expect(counterTest).toBe(3); }); test.afterEach(async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest === 1 || counterTest === 2).toBe(true); + expect(counterTest === 2 || counterTest === 3).toBe(true); }); test.afterAll(async ({}) => { expect(counterWorker).toBe(1); - expect(counterTest).toBe(2); + expect(counterTest).toBe(4); }); ` }); @@ -340,6 +340,35 @@ test('automatic fixtures should work', async ({ runInlineTest }) => { expect(result.results.map(r => r.status)).toEqual(['passed', 'passed']); }); +test('automatic fixture should start before regular fixture and teardown after', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const test = pwt.test; + test.use({ + auto: [ async ({}, runTest) => { + console.log('\\n%%auto-setup'); + await runTest(); + console.log('\\n%%auto-teardown'); + }, { auto: true } ], + foo: async ({}, runTest) => { + console.log('\\n%%foo-setup'); + await runTest(); + console.log('\\n%%foo-teardown'); + }, + }); + test('test 1', async ({ foo }) => { + }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%auto-setup', + '%%foo-setup', + '%%foo-teardown', + '%%auto-teardown', + ]); +}); + test('automatic fixtures should keep workerInfo after conditional skip', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.js': ` @@ -628,3 +657,24 @@ test('should run tests in order', async ({ runInlineTest }) => { '%%test3', ]); }); + +test('worker fixture should not receive TestInfo', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const test = pwt.test; + test.use({ + worker: [async ({}, use, info) => { + expect(info.title).toBe(undefined); + await use(); + }, { scope: 'worker' }], + test: async ({ worker }, use, info) => { + expect(info.title).not.toBe(undefined); + await use(); + }, + }); + test('test 1', async ({ test }) => { + }); + ` + }); + expect(result.exitCode).toBe(0); +}); diff --git a/tests/playwright-test/hooks.spec.ts b/tests/playwright-test/hooks.spec.ts index b806223dae..18bb26be72 100644 --- a/tests/playwright-test/hooks.spec.ts +++ b/tests/playwright-test/hooks.spec.ts @@ -20,6 +20,7 @@ test('hooks should work with fixtures', async ({ runInlineTest }) => { const { results } = await runInlineTest({ 'helper.ts': ` global.logs = []; + let counter = 0; export const test = pwt.test.extend({ w: [ async ({}, run) => { global.logs.push('+w'); @@ -29,7 +30,8 @@ test('hooks should work with fixtures', async ({ runInlineTest }) => { t: async ({}, run) => { global.logs.push('+t'); - await run(42); + await run(42 + counter); + ++counter; global.logs.push('-t'); }, }); @@ -37,36 +39,39 @@ test('hooks should work with fixtures', async ({ runInlineTest }) => { 'a.test.js': ` const { test } = require('./helper'); test.describe('suite', () => { - test.beforeAll(async ({ w }) => { - global.logs.push('beforeAll-' + w); + test.beforeAll(async ({ w, t }) => { + global.logs.push('beforeAll-' + w + '-' + t); }); - test.afterAll(async ({ w }) => { - global.logs.push('afterAll-' + w); + test.afterAll(async ({ w, t }) => { + global.logs.push('afterAll-' + w + '-' + t); }); - test.beforeEach(async ({t}) => { - global.logs.push('beforeEach-' + t); + test.beforeEach(async ({ w, t }) => { + global.logs.push('beforeEach-' + w + '-' + t); }); - test.afterEach(async ({t}) => { - global.logs.push('afterEach-' + t); + test.afterEach(async ({ w, t }) => { + global.logs.push('afterEach-' + w + '-' + t); }); - test('one', async ({t}) => { - global.logs.push('test'); - expect(t).toBe(42); + test('one', async ({ w, t }) => { + global.logs.push('test-' + w + '-' + t); }); }); - test('two', async ({t}) => { + test('two', async ({ t }) => { expect(global.logs).toEqual([ '+w', - 'beforeAll-17', '+t', - 'beforeEach-42', - 'test', - 'afterEach-42', + 'beforeAll-17-42', + '-t', + '+t', + 'beforeEach-17-43', + 'test-17-43', + 'afterEach-17-43', + '-t', + '+t', + 'afterAll-17-44', '-t', - 'afterAll-17', '+t', ]); }); @@ -214,3 +219,26 @@ test('beforeAll hooks are skipped when no tests in the suite are run', async ({ expect(result.output).toContain('%%beforeAll2'); expect(result.output).not.toContain('%%beforeAll1'); }); + +test('beforeAll hook should get retry index of the first test', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test.beforeAll(({}, testInfo) => { + console.log('\\n%%beforeall-retry-' + testInfo.retry); + }); + test('passed', ({}, testInfo) => { + console.log('\\n%%test-retry-' + testInfo.retry); + expect(testInfo.retry).toBe(1); + }); + `, + }, { retries: 1 }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([ + '%%beforeall-retry-0', + '%%test-retry-0', + '%%beforeall-retry-1', + '%%test-retry-1', + ]); +}); diff --git a/tests/playwright-test/types-2.spec.ts b/tests/playwright-test/types-2.spec.ts index bdf8131ba9..a0b8387185 100644 --- a/tests/playwright-test/types-2.spec.ts +++ b/tests/playwright-test/types-2.spec.ts @@ -27,6 +27,8 @@ test('basics should work', async ({runTSC}) => { testInfo.annotations[0].type; }); }); + // @ts-expect-error + test.foo(); ` }); expect(result.exitCode).toBe(0); diff --git a/tests/playwright-test/types.spec.ts b/tests/playwright-test/types.spec.ts index 4970c7bfe9..89cdc7a1b2 100644 --- a/tests/playwright-test/types.spec.ts +++ b/tests/playwright-test/types.spec.ts @@ -16,17 +16,6 @@ import { test, expect } from './playwright-test-fixtures'; -test('sanity', async ({runTSC}) => { - const result = await runTSC({ - 'a.spec.ts': ` - const { test } = pwt; - // @ts-expect-error - test.foo(); - ` - }); - expect(result.exitCode).toBe(0); -}); - test('should check types of fixtures', async ({runTSC}) => { const result = await runTSC({ 'helper.ts': ` @@ -125,9 +114,7 @@ test('should check types of fixtures', async ({runTSC}) => { // @ts-expect-error test.beforeAll(async ({ a }) => {}); - // @ts-expect-error test.beforeAll(async ({ foo, bar }) => {}); - test.beforeAll(async ({ bar }) => {}); test.beforeAll(() => {}); // @ts-expect-error @@ -137,9 +124,7 @@ test('should check types of fixtures', async ({runTSC}) => { // @ts-expect-error test.afterAll(async ({ a }) => {}); - // @ts-expect-error test.afterAll(async ({ foo, bar }) => {}); - test.afterAll(async ({ bar }) => {}); test.afterAll(() => {}); ` }); diff --git a/types/test.d.ts b/types/test.d.ts index e1bcec7b49..697d941555 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -2024,17 +2024,17 @@ export interface TestType Promise | any): void; + beforeAll(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; /** * Declares an `afterAll` hook that is executed once after all tests. When called in the scope of a test file, runs after * all tests in the file. When called inside a * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group, runs after all tests * in the group. - * @param hookFunction Hook function that takes one or two arguments: an object with fixtures and optional [WorkerInfo]. + * @param hookFunction Hook function that takes one or two arguments: an object with fixtures and optional [TestInfo]. */ - afterAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise | any): void; + afterAll(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; /** * Specifies parameters or fixtures to use in a single test file or a * [test.describe(title, callback)](https://playwright.dev/docs/api/class-test#test-describe) group. Most useful to diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 049aa6014d..534c6c63f1 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -247,8 +247,8 @@ export interface TestType Promise | any): void; afterEach(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; - beforeAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise | any): void; - afterAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise | any): void; + beforeAll(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; + afterAll(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void; step(title: string, body: () => Promise): Promise; expect: Expect;