diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index ba6bfde7ec..7b2aa39955 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -149,6 +149,18 @@ const config: PlaywrightTestConfig = { export default config; ``` +## property: TestConfig.globalScripts +* since: v1.30 +- type: ?<[string]|[RegExp]|[Array]<[string]|[RegExp]>> + +Files that contain global setup/teardown hooks. + +**Details** + +[`method: Test.beforeAll`] hooks in the matching files will run before testing starts. [`method: Test.afterAll`] hooks in the matching files will run after testing finishes. + +If global setup fails, test execution will be skipped. [`method: Test.afterAll`] hooks will run in the same worker process as [`method: Test.beforeAll`]. + ## property: TestConfig.globalSetup * since: v1.10 - type: ?<[string]> diff --git a/packages/playwright-test/src/dispatcher.ts b/packages/playwright-test/src/dispatcher.ts index 6f44cd8c3d..6f10231c25 100644 --- a/packages/playwright-test/src/dispatcher.ts +++ b/packages/playwright-test/src/dispatcher.ts @@ -32,7 +32,7 @@ export type TestGroup = { projectId: string; tests: TestCase[]; watchMode: boolean; - isProjectSetup: boolean; + phase: 'test' | 'projectSetup' | 'globalSetup'; }; type TestResultData = { @@ -573,7 +573,7 @@ class Worker extends EventEmitter { return { testId: test.id, retry: test.results.length }; }), watchMode: testGroup.watchMode, - projectSetup: testGroup.isProjectSetup, + phase: testGroup.phase, }; this.send({ method: 'run', params: runPayload }); } diff --git a/packages/playwright-test/src/ipc.ts b/packages/playwright-test/src/ipc.ts index e5b265fddd..ab776a1eef 100644 --- a/packages/playwright-test/src/ipc.ts +++ b/packages/playwright-test/src/ipc.ts @@ -95,7 +95,7 @@ export type RunPayload = { file: string; entries: TestEntry[]; watchMode: boolean; - projectSetup: boolean; + phase: 'test' | 'projectSetup' | 'globalSetup'; }; export type DonePayload = { diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 1906a85c29..dfd37d7ddd 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -138,6 +138,7 @@ export class Loader { this._fullConfig.shard = takeFirst(config.shard, baseFullConfig.shard); this._fullConfig._ignoreSnapshots = takeFirst(config.ignoreSnapshots, baseFullConfig._ignoreSnapshots); this._fullConfig.updateSnapshots = takeFirst(config.updateSnapshots, baseFullConfig.updateSnapshots); + this._fullConfig._globalScripts = takeFirst(config.globalScripts, null); const workers = takeFirst(config.workers, '50%'); if (typeof workers === 'string') { @@ -161,8 +162,9 @@ export class Loader { this._fullConfig._webServers = [webServers]; } this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata); + this._fullConfig._globalProject = this._resolveProject(config, this._fullConfig, globalScriptsProject, throwawayArtifactsPath); this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath)); - this._assignUniqueProjectIds(this._fullConfig.projects); + this._assignUniqueProjectIds([...this._fullConfig.projects, this._fullConfig._globalProject]); } private _assignUniqueProjectIds(projects: FullProjectInternal[]) { @@ -180,12 +182,12 @@ export class Loader { } } - async loadTestFile(file: string, environment: 'runner' | 'worker', projectSetup: boolean) { + async loadTestFile(file: string, environment: 'runner' | 'worker', phase: 'test' | 'projectSetup' | 'globalSetup') { if (cachedFileSuites.has(file)) return cachedFileSuites.get(file)!; const suite = new Suite(path.relative(this._fullConfig.rootDir, file) || path.basename(file), 'file'); suite._requireFile = file; - suite._isProjectSetup = projectSetup; + suite._phase = phase; suite.location = { file, line: 0, column: 0 }; setCurrentlyLoadingFileSuite(suite); @@ -639,6 +641,11 @@ function validateProject(file: string, project: Project, title: string) { } } +const globalScriptsProject: Project = { + name: 'Global Scripts', + repeatEach: 1, +}; + export const baseFullConfig: FullConfigInternal = { forbidOnly: false, fullyParallel: false, @@ -668,6 +675,8 @@ export const baseFullConfig: FullConfigInternal = { _maxConcurrentTestGroups: 0, _ignoreSnapshots: false, _workerIsolation: 'isolate-pools', + _globalScripts: null, + _globalProject: { } as FullProjectInternal, }; function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined { diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index be256ca61d..0c91dd6830 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -238,13 +238,29 @@ export class Runner { return projects; } - private async _collectFiles(projects: FullProjectInternal[], commandLineFileFilters: TestFileFilter[]): Promise<{filesByProject: Map; setupFiles: Set}> { + private async _collectFiles(projects: FullProjectInternal[], commandLineFileFilters: TestFileFilter[]): Promise<{filesByProject: Map; setupFiles: Set, globalSetupFiles: Set}> { const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx']; const testFileExtension = (file: string) => extensions.includes(path.extname(file)); const filesByProject = new Map(); const setupFiles = new Set(); const fileToProjectName = new Map(); const commandLineFileMatcher = commandLineFileFilters.length ? createFileMatcherFromFilters(commandLineFileFilters) : () => true; + + const config = this._loader.fullConfig(); + const globalSetupFiles = new Set(); + if (config._globalScripts) { + const allFiles = await collectFiles(config.rootDir, true); + const globalScriptMatch = createFileMatcher(config._globalScripts); + const globalScripts = allFiles.filter(file => { + if (!testFileExtension(file) || !globalScriptMatch(file)) + return false; + fileToProjectName.set(file, config._globalProject.name); + globalSetupFiles.add(file); + return true; + }); + filesByProject.set(config._globalProject, globalScripts); + } + for (const project of projects) { const allFiles = await collectFiles(project.testDir, project._respectGitIgnore); const setupMatch = createFileMatcher(project._setupMatch); @@ -275,42 +291,45 @@ export class Runner { filesByProject.set(project, testFiles); } - return { filesByProject, setupFiles }; + return { filesByProject, setupFiles, globalSetupFiles }; } - private async _collectTestGroups(options: RunOptions): Promise<{ rootSuite: Suite, projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> { + private async _collectTestGroups(options: RunOptions): Promise<{ rootSuite: Suite, globalSetupGroups: TestGroup[], projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> { const config = this._loader.fullConfig(); const projects = this._collectProjects(options.projectFilter); - const { filesByProject, setupFiles } = await this._collectFiles(projects, options.testFileFilters); + const { filesByProject, setupFiles, globalSetupFiles } = await this._collectFiles(projects, options.testFileFilters); - let result = await this._createFilteredRootSuite(options, filesByProject, new Set(), !!setupFiles.size, setupFiles); + let result = await this._createFilteredRootSuite(options, filesByProject, new Set(), !!setupFiles.size, setupFiles, globalSetupFiles); if (setupFiles.size) { const allTests = result.rootSuite.allTests(); - const tests = allTests.filter(test => !test._isProjectSetup); + const tests = allTests.filter(test => test._phase === 'test'); // If >0 tests match and // - none of the setup files match the filter then we run all setup files, // - if the filter also matches some of the setup tests, we'll run only // that maching subset of setup tests. if (tests.length > 0 && tests.length === allTests.length) - result = await this._createFilteredRootSuite(options, filesByProject, setupFiles, false, setupFiles); + result = await this._createFilteredRootSuite(options, filesByProject, setupFiles, false, setupFiles, globalSetupFiles); } this._fatalErrors.push(...result.fatalErrors); const { rootSuite } = result; const allTestGroups = createTestGroups(rootSuite.suites, config.workers); + const globalSetupGroups = []; const projectSetupGroups = []; const testGroups = []; for (const group of allTestGroups) { - if (group.isProjectSetup) + if (group.phase === 'projectSetup') projectSetupGroups.push(group); + else if (group.phase === 'globalSetup') + globalSetupGroups.push(group); else testGroups.push(group); } - return { rootSuite, projectSetupGroups, testGroups }; + return { rootSuite, globalSetupGroups, projectSetupGroups, testGroups }; } - private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map, doNotFilterFiles: Set, shouldCloneTests: boolean, setupFiles: Set): Promise<{rootSuite: Suite, fatalErrors: TestError[]}> { + private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map, doNotFilterFiles: Set, shouldCloneTests: boolean, setupFiles: Set, globalSetupFiles: Set): Promise<{rootSuite: Suite, fatalErrors: TestError[]}> { const config = this._loader.fullConfig(); const fatalErrors: TestError[] = []; const allTestFiles = new Set(); @@ -320,7 +339,12 @@ export class Runner { // Add all tests. const preprocessRoot = new Suite('', 'root'); for (const file of allTestFiles) { - const fileSuite = await this._loader.loadTestFile(file, 'runner', setupFiles.has(file)); + let type: 'test' | 'projectSetup' | 'globalSetup' = 'test'; + if (globalSetupFiles.has(file)) + type = 'globalSetup'; + else if (setupFiles.has(file)) + type = 'projectSetup'; + const fileSuite = await this._loader.loadTestFile(file, 'runner', type); if (fileSuite._loadError) fatalErrors.push(fileSuite._loadError); // We have to clone only if there maybe subsequent calls of this method. @@ -439,7 +463,10 @@ export class Runner { rootSuite.suites = []; rootSuite.tests = []; } else { - filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test)); + // Unlike project setup files global setup always run regardless of the selected tests. + // Because of that we don't add global setup entries to shardTests to avoid running empty + // shards which have only global setup. + filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test) || test._phase === 'globalSetup'); } } @@ -447,7 +474,7 @@ export class Runner { const config = this._loader.fullConfig(); // Each entry is an array of test groups that can be run concurrently. All // test groups from the previos entries must finish before entry starts. - const { rootSuite, projectSetupGroups, testGroups } = await this._collectTestGroups(options); + const { rootSuite, globalSetupGroups, projectSetupGroups, testGroups } = await this._collectTestGroups(options); // Fail when no tests. if (!rootSuite.allTests().length && !options.passWithNoTests) @@ -455,7 +482,7 @@ export class Runner { this._filterForCurrentShard(rootSuite, projectSetupGroups, testGroups); - config._maxConcurrentTestGroups = Math.max(projectSetupGroups.length, testGroups.length); + config._maxConcurrentTestGroups = Math.max(globalSetupGroups.length, projectSetupGroups.length, testGroups.length); // Report begin this._reporter.onBegin?.(config, rootSuite); @@ -492,15 +519,23 @@ export class Runner { // Run tests. try { - let dispatchResult = await this._dispatchToWorkers(projectSetupGroups); + // TODO: run only setups, keep workers alive, inherit process.env from global setup workers + let dispatchResult = await this._dispatchToWorkers(globalSetupGroups); if (dispatchResult === 'success') { - const failedSetupProjectIds = new Set(); - for (const testGroup of projectSetupGroups) { - if (testGroup.tests.some(test => !test.ok())) - failedSetupProjectIds.add(testGroup.projectId); + if (globalSetupGroups.some(group => group.tests.some(test => !test.ok()))) { + this._skipTestsFromMatchingGroups([...testGroups, ...projectSetupGroups], () => true); + } else { + dispatchResult = await this._dispatchToWorkers(projectSetupGroups); + if (dispatchResult === 'success') { + const failedSetupProjectIds = new Set(); + for (const testGroup of projectSetupGroups) { + if (testGroup.tests.some(test => !test.ok())) + failedSetupProjectIds.add(testGroup.projectId); + } + const testGroupsToRun = this._skipTestsFromMatchingGroups(testGroups, group => failedSetupProjectIds.has(group.projectId)); + dispatchResult = await this._dispatchToWorkers(testGroupsToRun); + } } - const testGroupsToRun = this._skipTestsFromFailedProjects(testGroups, failedSetupProjectIds); - dispatchResult = await this._dispatchToWorkers(testGroupsToRun); } if (dispatchResult === 'signal') { result.status = 'interrupted'; @@ -534,10 +569,10 @@ export class Runner { return 'success'; } - private _skipTestsFromFailedProjects(testGroups: TestGroup[], failedProjects: Set): TestGroup[] { + private _skipTestsFromMatchingGroups(testGroups: TestGroup[], groupFilter: (g: TestGroup) => boolean): TestGroup[] { const result = []; for (const group of testGroups) { - if (failedProjects.has(group.projectId)) { + if (groupFilter(group)) { for (const test of group.tests) { const result = test._appendTestResult(); this._reporter.onTestBegin?.(test, result); @@ -823,7 +858,7 @@ function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[] projectId: test._projectId, tests: [], watchMode: false, - isProjectSetup: test._isProjectSetup, + phase: test._phase, }; }; diff --git a/packages/playwright-test/src/test.ts b/packages/playwright-test/src/test.ts index cb490f4edf..9f5f87a39f 100644 --- a/packages/playwright-test/src/test.ts +++ b/packages/playwright-test/src/test.ts @@ -23,7 +23,7 @@ class Base { title: string; _only = false; _requireFile: string = ''; - _isProjectSetup: boolean = false; + _phase: 'test' | 'projectSetup' | 'globalSetup' = 'test'; constructor(title: string) { this.title = title; @@ -121,7 +121,7 @@ export class Suite extends Base implements reporterTypes.Suite { suite._only = this._only; suite.location = this.location; suite._requireFile = this._requireFile; - suite._isProjectSetup = this._isProjectSetup; + suite._phase = this._phase; suite._use = this._use.slice(); suite._hooks = this._hooks.slice(); suite._timeout = this._timeout; @@ -193,7 +193,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { const test = new TestCase(this.title, this.fn, this._testType, this.location); test._only = this._only; test._requireFile = this._requireFile; - test._isProjectSetup = this._isProjectSetup; + test._phase = this._phase; test.expectedStatus = this.expectedStatus; test.annotations = this.annotations.slice(); test._annotateWithInheritence = this._annotateWithInheritence; diff --git a/packages/playwright-test/src/testType.ts b/packages/playwright-test/src/testType.ts index 538d622d11..2b21659a21 100644 --- a/packages/playwright-test/src/testType.ts +++ b/packages/playwright-test/src/testType.ts @@ -80,10 +80,10 @@ export class TestTypeImpl { ].join('\n'), location); return; } - if (allowedContext === 'projectSetup' && !suite._isProjectSetup) + if (allowedContext === 'projectSetup' && suite._phase !== 'projectSetup') addFatalError(`${title} is only allowed in a project setup file.`, location); - else if (allowedContext === 'test' && suite._isProjectSetup) - addFatalError(`${title} is not allowed in a project setup file.`, location); + else if (allowedContext === 'test' && suite._phase !== 'test' && suite._phase !== 'globalSetup') + addFatalError(`${title} is not allowed in a setup file.`, location); return suite; } @@ -106,7 +106,7 @@ export class TestTypeImpl { return; const test = new TestCase(title, fn, this, location); test._requireFile = suite._requireFile; - test._isProjectSetup = suite._isProjectSetup; + test._phase = suite._phase; suite._addTest(test); if (type === 'only' || type === 'projectSetupOnly') @@ -134,7 +134,7 @@ export class TestTypeImpl { const child = new Suite(title, 'describe'); child._requireFile = suite._requireFile; - child._isProjectSetup = suite._isProjectSetup; + child._phase = suite._phase; child.location = location; suite._addSuite(child); diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index a261ae2551..1a3e6fba95 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -54,6 +54,10 @@ export interface FullConfigInternal extends FullConfigPublic { */ webServer: FullConfigPublic['webServer']; _webServers: Exclude[]; + _globalScripts: string | RegExp | (string | RegExp)[] | null; + + // This is an ephemeral project that is not added to `projects` list below. + _globalProject: FullProjectInternal; // Overrides the public field. projects: FullProjectInternal[]; diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index e50e832ba9..9793776858 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -160,7 +160,11 @@ export class WorkerRunner extends EventEmitter { return; this._loader = await Loader.deserialize(this._params.loader); - this._project = this._loader.fullConfig().projects.find(p => p._id === this._params.projectId)!; + const globalProject = this._loader.fullConfig()._globalProject; + if (this._params.projectId === globalProject._id) + this._project = globalProject; + else + this._project = this._loader.fullConfig().projects.find(p => p._id === this._params.projectId)!; } async runTestGroup(runPayload: RunPayload) { @@ -169,7 +173,7 @@ export class WorkerRunner extends EventEmitter { let fatalUnknownTestIds; try { await this._loadIfNeeded(); - const fileSuite = await this._loader.loadTestFile(runPayload.file, 'worker', runPayload.projectSetup); + const fileSuite = await this._loader.loadTestFile(runPayload.file, 'worker', runPayload.phase); const suite = this._loader.buildFileSuiteForProject(this._project, fileSuite, this._params.repeatEachIndex, test => { if (runPayload.watchMode) { const testResolvedPayload: WatchTestResolvedPayload = { diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index ff8dec2c46..0223aee199 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -646,6 +646,22 @@ interface TestConfig { */ fullyParallel?: boolean; + /** + * Files that contain global setup/teardown hooks. + * + * **Details** + * + * [test.beforeAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-before-all) hooks in the matching + * files will run before testing starts. + * [test.afterAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-all) hooks in the matching + * files will run after testing finishes. + * + * If global setup fails, test execution will be skipped. + * [test.afterAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-all) hooks will run in the same + * worker process as [test.beforeAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-before-all). + */ + globalScripts?: string|RegExp|Array; + /** * Path to the global setup file. This file will be required and run before all the tests. It must export a single * function that takes a [`TestConfig`] argument. diff --git a/tests/playwright-test/global-scripts.spec.ts b/tests/playwright-test/global-scripts.spec.ts new file mode 100644 index 0000000000..6ca46b0926 --- /dev/null +++ b/tests/playwright-test/global-scripts.spec.ts @@ -0,0 +1,225 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import path from 'path'; +import { test, expect } from './playwright-test-fixtures'; + +type Timeline = { titlePath: string[], event: 'begin' | 'end' }[]; + +function formatTimeline(timeline: Timeline) { + return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n'); +} + +function formatFileNames(timeline: Timeline) { + return timeline.map(e => e.titlePath[2]).join('\n'); +} + +function fileNames(timeline: Timeline) { + const fileNames = Array.from(new Set(timeline.map(({ titlePath }) => { + const name = titlePath[2]; + const index = name.lastIndexOf(path.sep); + if (index === -1) + return name; + return name.slice(index + 1); + })).keys()); + fileNames.sort(); + return fileNames; +} + +function expectFilesRunBefore(timeline: Timeline, before: string[], after: string[]) { + const fileBegin = name => { + const index = timeline.findIndex(({ titlePath }) => titlePath[2] === name); + expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1); + return index; + }; + const fileEnd = name => { + // There is no Array.findLastIndex in Node < 18. + let index = -1; + for (index = timeline.length - 1; index >= 0; index--) { + if (timeline[index].titlePath[2] === name) + break; + } + expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1); + return index; + }; + + for (const b of before) { + const bEnd = fileEnd(b); + for (const a of after) { + const aBegin = fileBegin(a); + expect(bEnd < aBegin, `'${b}' expected to finish before ${a}, actual order:\n${formatTimeline(timeline)}`).toBeTruthy(); + } + } +} + +test('should work for one project', async ({ runGroups }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + globalScripts: /.*global.ts/, + projects: [ + { + name: 'p1', + testMatch: /.*.test.ts/, + }, + ] + };`, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'global.ts': ` + const { test } = pwt; + test('setup1', async () => { }); + test('setup2', async () => { }); + `, + }; + const { exitCode, passed, timeline } = await runGroups(files); + expect(exitCode).toBe(0); + expect(passed).toBe(4); + expect(formatTimeline(timeline)).toEqual(`Global Scripts > global.ts > setup1 [begin] +Global Scripts > global.ts > setup1 [end] +Global Scripts > global.ts > setup2 [begin] +Global Scripts > global.ts > setup2 [end] +p1 > a.test.ts > test1 [begin] +p1 > a.test.ts > test1 [end] +p1 > a.test.ts > test2 [begin] +p1 > a.test.ts > test2 [end]`); +}); + +test('should work for several projects', async ({ runGroups }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + globalScripts: /.*global.ts/, + projects: [ + { + name: 'p1', + testMatch: /.*a.test.ts/, + }, + { + name: 'p2', + testMatch: /.*b.test.ts/, + }, + ] + };`, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'b.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'global.ts': ` + const { test } = pwt; + test('setup1', async () => { }); + test('setup2', async () => { }); + `, + }; + const { exitCode, passed, timeline } = await runGroups(files); + expect(exitCode).toBe(0); + expect(passed).toBe(6); + expectFilesRunBefore(timeline, [`global.ts`], [`a.test.ts`, `b.test.ts`]); +}); + +test('should skip tests if global setup fails', async ({ runGroups }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + globalScripts: /.*global.ts/, + projects: [ + { + name: 'p1', + testMatch: /.*a.test.ts/, + }, + { + name: 'p2', + testMatch: /.*b.test.ts/, + }, + ] + };`, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'b.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + `, + 'global.ts': ` + const { test, expect } = pwt; + test('setup1', async () => { }); + test('setup2', async () => { expect(1).toBe(2) }); + `, + }; + const { exitCode, passed, skipped } = await runGroups(files); + expect(exitCode).toBe(1); + expect(passed).toBe(1); + expect(skipped).toBe(3); +}); + +test('should run setup in each project shard', async ({ runGroups }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + globalScripts: /.*global.ts/, + projects: [ + { + name: 'p1', + }, + ] + };`, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + test('test3', async () => { }); + test('test4', async () => { }); + `, + 'b.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'global.ts': ` + const { test, expect } = pwt; + test('setup1', async () => { }); + test('setup2', async () => { }); + `, + }; + + { // Shard 1/2 + const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '1/2' }); + expect(output).toContain('Running 6 tests using 1 worker, shard 1 of 2'); + expect(fileNames(timeline)).toEqual(['a.test.ts', 'global.ts']); + expectFilesRunBefore(timeline, [`global.ts`], [`a.test.ts`]); + expect(exitCode).toBe(0); + expect(passed).toBe(6); + } + { // Shard 2/2 + const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '2/2' }); + expect(output).toContain('Running 4 tests using 1 worker, shard 2 of 2'); + expect(fileNames(timeline)).toEqual(['b.test.ts', 'global.ts']); + expectFilesRunBefore(timeline, [`global.ts`], [`b.test.ts`]); + expect(exitCode).toBe(0); + expect(passed).toBe(4); + } +}); + diff --git a/tests/playwright-test/project-setup.spec.ts b/tests/playwright-test/project-setup.spec.ts index 8b747219e3..4166c634cc 100644 --- a/tests/playwright-test/project-setup.spec.ts +++ b/tests/playwright-test/project-setup.spec.ts @@ -892,7 +892,7 @@ test('should prohibit beforeAll hooks in setup files', async ({ runGroups }, tes const { exitCode, output } = await runGroups(files); expect(exitCode).toBe(1); - expect(output).toContain('test.beforeAll() is not allowed in a project setup file'); + expect(output).toContain('test.beforeAll() is not allowed in a setup file'); }); test('should prohibit test in setup files', async ({ runGroups }, testInfo) => { @@ -914,7 +914,7 @@ test('should prohibit test in setup files', async ({ runGroups }, testInfo) => { const { exitCode, output } = await runGroups(files); expect(exitCode).toBe(1); - expect(output).toContain('test() is not allowed in a project setup file'); + expect(output).toContain('test() is not allowed in a setup file'); }); test('should prohibit test hooks in setup files', async ({ runGroups }, testInfo) => { @@ -936,5 +936,5 @@ test('should prohibit test hooks in setup files', async ({ runGroups }, testInfo const { exitCode, output } = await runGroups(files); expect(exitCode).toBe(1); - expect(output).toContain('test.beforeEach() is not allowed in a project setup file'); + expect(output).toContain('test.beforeEach() is not allowed in a setup file'); });