From 730a197c8018a891be821da7100af0a8f63e10ea Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 12 Jan 2023 13:02:54 -0800 Subject: [PATCH] feat: config.globalScripts (#20062) Introduce config.globalScripts. Tests from the matching files will run before all projects. We'll only allow beforeAll/afterAll instead of tests in such files (next PR). Global scripts are executed as part of 'Global Scripts' project which is not present in FullConfig.projects but may be referenced by corresponding global setup Suites. Signed-off-by: Yury Semikhatsky Co-authored-by: Dmitry Gozman --- docs/src/test-api/class-testconfig.md | 12 + packages/playwright-test/src/dispatcher.ts | 4 +- packages/playwright-test/src/ipc.ts | 2 +- packages/playwright-test/src/loader.ts | 15 +- packages/playwright-test/src/runner.ts | 83 +++++-- packages/playwright-test/src/test.ts | 6 +- packages/playwright-test/src/testType.ts | 10 +- packages/playwright-test/src/types.ts | 4 + packages/playwright-test/src/workerRunner.ts | 8 +- packages/playwright-test/types/test.d.ts | 16 ++ tests/playwright-test/global-scripts.spec.ts | 225 +++++++++++++++++++ tests/playwright-test/project-setup.spec.ts | 6 +- 12 files changed, 348 insertions(+), 43 deletions(-) create mode 100644 tests/playwright-test/global-scripts.spec.ts diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index ba6bfde7ec..7b2aa39955 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -149,6 +149,18 @@ const config: PlaywrightTestConfig = { export default config; ``` +## property: TestConfig.globalScripts +* since: v1.30 +- type: ?<[string]|[RegExp]|[Array]<[string]|[RegExp]>> + +Files that contain global setup/teardown hooks. + +**Details** + +[`method: Test.beforeAll`] hooks in the matching files will run before testing starts. [`method: Test.afterAll`] hooks in the matching files will run after testing finishes. + +If global setup fails, test execution will be skipped. [`method: Test.afterAll`] hooks will run in the same worker process as [`method: Test.beforeAll`]. + ## property: TestConfig.globalSetup * since: v1.10 - type: ?<[string]> diff --git a/packages/playwright-test/src/dispatcher.ts b/packages/playwright-test/src/dispatcher.ts index 6f44cd8c3d..6f10231c25 100644 --- a/packages/playwright-test/src/dispatcher.ts +++ b/packages/playwright-test/src/dispatcher.ts @@ -32,7 +32,7 @@ export type TestGroup = { projectId: string; tests: TestCase[]; watchMode: boolean; - isProjectSetup: boolean; + phase: 'test' | 'projectSetup' | 'globalSetup'; }; type TestResultData = { @@ -573,7 +573,7 @@ class Worker extends EventEmitter { return { testId: test.id, retry: test.results.length }; }), watchMode: testGroup.watchMode, - projectSetup: testGroup.isProjectSetup, + phase: testGroup.phase, }; this.send({ method: 'run', params: runPayload }); } diff --git a/packages/playwright-test/src/ipc.ts b/packages/playwright-test/src/ipc.ts index e5b265fddd..ab776a1eef 100644 --- a/packages/playwright-test/src/ipc.ts +++ b/packages/playwright-test/src/ipc.ts @@ -95,7 +95,7 @@ export type RunPayload = { file: string; entries: TestEntry[]; watchMode: boolean; - projectSetup: boolean; + phase: 'test' | 'projectSetup' | 'globalSetup'; }; export type DonePayload = { diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 1906a85c29..dfd37d7ddd 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -138,6 +138,7 @@ export class Loader { this._fullConfig.shard = takeFirst(config.shard, baseFullConfig.shard); this._fullConfig._ignoreSnapshots = takeFirst(config.ignoreSnapshots, baseFullConfig._ignoreSnapshots); this._fullConfig.updateSnapshots = takeFirst(config.updateSnapshots, baseFullConfig.updateSnapshots); + this._fullConfig._globalScripts = takeFirst(config.globalScripts, null); const workers = takeFirst(config.workers, '50%'); if (typeof workers === 'string') { @@ -161,8 +162,9 @@ export class Loader { this._fullConfig._webServers = [webServers]; } this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata); + this._fullConfig._globalProject = this._resolveProject(config, this._fullConfig, globalScriptsProject, throwawayArtifactsPath); this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath)); - this._assignUniqueProjectIds(this._fullConfig.projects); + this._assignUniqueProjectIds([...this._fullConfig.projects, this._fullConfig._globalProject]); } private _assignUniqueProjectIds(projects: FullProjectInternal[]) { @@ -180,12 +182,12 @@ export class Loader { } } - async loadTestFile(file: string, environment: 'runner' | 'worker', projectSetup: boolean) { + async loadTestFile(file: string, environment: 'runner' | 'worker', phase: 'test' | 'projectSetup' | 'globalSetup') { if (cachedFileSuites.has(file)) return cachedFileSuites.get(file)!; const suite = new Suite(path.relative(this._fullConfig.rootDir, file) || path.basename(file), 'file'); suite._requireFile = file; - suite._isProjectSetup = projectSetup; + suite._phase = phase; suite.location = { file, line: 0, column: 0 }; setCurrentlyLoadingFileSuite(suite); @@ -639,6 +641,11 @@ function validateProject(file: string, project: Project, title: string) { } } +const globalScriptsProject: Project = { + name: 'Global Scripts', + repeatEach: 1, +}; + export const baseFullConfig: FullConfigInternal = { forbidOnly: false, fullyParallel: false, @@ -668,6 +675,8 @@ export const baseFullConfig: FullConfigInternal = { _maxConcurrentTestGroups: 0, _ignoreSnapshots: false, _workerIsolation: 'isolate-pools', + _globalScripts: null, + _globalProject: { } as FullProjectInternal, }; function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined { diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index be256ca61d..0c91dd6830 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -238,13 +238,29 @@ export class Runner { return projects; } - private async _collectFiles(projects: FullProjectInternal[], commandLineFileFilters: TestFileFilter[]): Promise<{filesByProject: Map; setupFiles: Set}> { + private async _collectFiles(projects: FullProjectInternal[], commandLineFileFilters: TestFileFilter[]): Promise<{filesByProject: Map; setupFiles: Set, globalSetupFiles: Set}> { const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx']; const testFileExtension = (file: string) => extensions.includes(path.extname(file)); const filesByProject = new Map(); const setupFiles = new Set(); const fileToProjectName = new Map(); const commandLineFileMatcher = commandLineFileFilters.length ? createFileMatcherFromFilters(commandLineFileFilters) : () => true; + + const config = this._loader.fullConfig(); + const globalSetupFiles = new Set(); + if (config._globalScripts) { + const allFiles = await collectFiles(config.rootDir, true); + const globalScriptMatch = createFileMatcher(config._globalScripts); + const globalScripts = allFiles.filter(file => { + if (!testFileExtension(file) || !globalScriptMatch(file)) + return false; + fileToProjectName.set(file, config._globalProject.name); + globalSetupFiles.add(file); + return true; + }); + filesByProject.set(config._globalProject, globalScripts); + } + for (const project of projects) { const allFiles = await collectFiles(project.testDir, project._respectGitIgnore); const setupMatch = createFileMatcher(project._setupMatch); @@ -275,42 +291,45 @@ export class Runner { filesByProject.set(project, testFiles); } - return { filesByProject, setupFiles }; + return { filesByProject, setupFiles, globalSetupFiles }; } - private async _collectTestGroups(options: RunOptions): Promise<{ rootSuite: Suite, projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> { + private async _collectTestGroups(options: RunOptions): Promise<{ rootSuite: Suite, globalSetupGroups: TestGroup[], projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> { const config = this._loader.fullConfig(); const projects = this._collectProjects(options.projectFilter); - const { filesByProject, setupFiles } = await this._collectFiles(projects, options.testFileFilters); + const { filesByProject, setupFiles, globalSetupFiles } = await this._collectFiles(projects, options.testFileFilters); - let result = await this._createFilteredRootSuite(options, filesByProject, new Set(), !!setupFiles.size, setupFiles); + let result = await this._createFilteredRootSuite(options, filesByProject, new Set(), !!setupFiles.size, setupFiles, globalSetupFiles); if (setupFiles.size) { const allTests = result.rootSuite.allTests(); - const tests = allTests.filter(test => !test._isProjectSetup); + const tests = allTests.filter(test => test._phase === 'test'); // If >0 tests match and // - none of the setup files match the filter then we run all setup files, // - if the filter also matches some of the setup tests, we'll run only // that maching subset of setup tests. if (tests.length > 0 && tests.length === allTests.length) - result = await this._createFilteredRootSuite(options, filesByProject, setupFiles, false, setupFiles); + result = await this._createFilteredRootSuite(options, filesByProject, setupFiles, false, setupFiles, globalSetupFiles); } this._fatalErrors.push(...result.fatalErrors); const { rootSuite } = result; const allTestGroups = createTestGroups(rootSuite.suites, config.workers); + const globalSetupGroups = []; const projectSetupGroups = []; const testGroups = []; for (const group of allTestGroups) { - if (group.isProjectSetup) + if (group.phase === 'projectSetup') projectSetupGroups.push(group); + else if (group.phase === 'globalSetup') + globalSetupGroups.push(group); else testGroups.push(group); } - return { rootSuite, projectSetupGroups, testGroups }; + return { rootSuite, globalSetupGroups, projectSetupGroups, testGroups }; } - private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map, doNotFilterFiles: Set, shouldCloneTests: boolean, setupFiles: Set): Promise<{rootSuite: Suite, fatalErrors: TestError[]}> { + private async _createFilteredRootSuite(options: RunOptions, filesByProject: Map, doNotFilterFiles: Set, shouldCloneTests: boolean, setupFiles: Set, globalSetupFiles: Set): Promise<{rootSuite: Suite, fatalErrors: TestError[]}> { const config = this._loader.fullConfig(); const fatalErrors: TestError[] = []; const allTestFiles = new Set(); @@ -320,7 +339,12 @@ export class Runner { // Add all tests. const preprocessRoot = new Suite('', 'root'); for (const file of allTestFiles) { - const fileSuite = await this._loader.loadTestFile(file, 'runner', setupFiles.has(file)); + let type: 'test' | 'projectSetup' | 'globalSetup' = 'test'; + if (globalSetupFiles.has(file)) + type = 'globalSetup'; + else if (setupFiles.has(file)) + type = 'projectSetup'; + const fileSuite = await this._loader.loadTestFile(file, 'runner', type); if (fileSuite._loadError) fatalErrors.push(fileSuite._loadError); // We have to clone only if there maybe subsequent calls of this method. @@ -439,7 +463,10 @@ export class Runner { rootSuite.suites = []; rootSuite.tests = []; } else { - filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test)); + // Unlike project setup files global setup always run regardless of the selected tests. + // Because of that we don't add global setup entries to shardTests to avoid running empty + // shards which have only global setup. + filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test) || test._phase === 'globalSetup'); } } @@ -447,7 +474,7 @@ export class Runner { const config = this._loader.fullConfig(); // Each entry is an array of test groups that can be run concurrently. All // test groups from the previos entries must finish before entry starts. - const { rootSuite, projectSetupGroups, testGroups } = await this._collectTestGroups(options); + const { rootSuite, globalSetupGroups, projectSetupGroups, testGroups } = await this._collectTestGroups(options); // Fail when no tests. if (!rootSuite.allTests().length && !options.passWithNoTests) @@ -455,7 +482,7 @@ export class Runner { this._filterForCurrentShard(rootSuite, projectSetupGroups, testGroups); - config._maxConcurrentTestGroups = Math.max(projectSetupGroups.length, testGroups.length); + config._maxConcurrentTestGroups = Math.max(globalSetupGroups.length, projectSetupGroups.length, testGroups.length); // Report begin this._reporter.onBegin?.(config, rootSuite); @@ -492,15 +519,23 @@ export class Runner { // Run tests. try { - let dispatchResult = await this._dispatchToWorkers(projectSetupGroups); + // TODO: run only setups, keep workers alive, inherit process.env from global setup workers + let dispatchResult = await this._dispatchToWorkers(globalSetupGroups); if (dispatchResult === 'success') { - const failedSetupProjectIds = new Set(); - for (const testGroup of projectSetupGroups) { - if (testGroup.tests.some(test => !test.ok())) - failedSetupProjectIds.add(testGroup.projectId); + if (globalSetupGroups.some(group => group.tests.some(test => !test.ok()))) { + this._skipTestsFromMatchingGroups([...testGroups, ...projectSetupGroups], () => true); + } else { + dispatchResult = await this._dispatchToWorkers(projectSetupGroups); + if (dispatchResult === 'success') { + const failedSetupProjectIds = new Set(); + for (const testGroup of projectSetupGroups) { + if (testGroup.tests.some(test => !test.ok())) + failedSetupProjectIds.add(testGroup.projectId); + } + const testGroupsToRun = this._skipTestsFromMatchingGroups(testGroups, group => failedSetupProjectIds.has(group.projectId)); + dispatchResult = await this._dispatchToWorkers(testGroupsToRun); + } } - const testGroupsToRun = this._skipTestsFromFailedProjects(testGroups, failedSetupProjectIds); - dispatchResult = await this._dispatchToWorkers(testGroupsToRun); } if (dispatchResult === 'signal') { result.status = 'interrupted'; @@ -534,10 +569,10 @@ export class Runner { return 'success'; } - private _skipTestsFromFailedProjects(testGroups: TestGroup[], failedProjects: Set): TestGroup[] { + private _skipTestsFromMatchingGroups(testGroups: TestGroup[], groupFilter: (g: TestGroup) => boolean): TestGroup[] { const result = []; for (const group of testGroups) { - if (failedProjects.has(group.projectId)) { + if (groupFilter(group)) { for (const test of group.tests) { const result = test._appendTestResult(); this._reporter.onTestBegin?.(test, result); @@ -823,7 +858,7 @@ function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[] projectId: test._projectId, tests: [], watchMode: false, - isProjectSetup: test._isProjectSetup, + phase: test._phase, }; }; diff --git a/packages/playwright-test/src/test.ts b/packages/playwright-test/src/test.ts index cb490f4edf..9f5f87a39f 100644 --- a/packages/playwright-test/src/test.ts +++ b/packages/playwright-test/src/test.ts @@ -23,7 +23,7 @@ class Base { title: string; _only = false; _requireFile: string = ''; - _isProjectSetup: boolean = false; + _phase: 'test' | 'projectSetup' | 'globalSetup' = 'test'; constructor(title: string) { this.title = title; @@ -121,7 +121,7 @@ export class Suite extends Base implements reporterTypes.Suite { suite._only = this._only; suite.location = this.location; suite._requireFile = this._requireFile; - suite._isProjectSetup = this._isProjectSetup; + suite._phase = this._phase; suite._use = this._use.slice(); suite._hooks = this._hooks.slice(); suite._timeout = this._timeout; @@ -193,7 +193,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { const test = new TestCase(this.title, this.fn, this._testType, this.location); test._only = this._only; test._requireFile = this._requireFile; - test._isProjectSetup = this._isProjectSetup; + test._phase = this._phase; test.expectedStatus = this.expectedStatus; test.annotations = this.annotations.slice(); test._annotateWithInheritence = this._annotateWithInheritence; diff --git a/packages/playwright-test/src/testType.ts b/packages/playwright-test/src/testType.ts index 538d622d11..2b21659a21 100644 --- a/packages/playwright-test/src/testType.ts +++ b/packages/playwright-test/src/testType.ts @@ -80,10 +80,10 @@ export class TestTypeImpl { ].join('\n'), location); return; } - if (allowedContext === 'projectSetup' && !suite._isProjectSetup) + if (allowedContext === 'projectSetup' && suite._phase !== 'projectSetup') addFatalError(`${title} is only allowed in a project setup file.`, location); - else if (allowedContext === 'test' && suite._isProjectSetup) - addFatalError(`${title} is not allowed in a project setup file.`, location); + else if (allowedContext === 'test' && suite._phase !== 'test' && suite._phase !== 'globalSetup') + addFatalError(`${title} is not allowed in a setup file.`, location); return suite; } @@ -106,7 +106,7 @@ export class TestTypeImpl { return; const test = new TestCase(title, fn, this, location); test._requireFile = suite._requireFile; - test._isProjectSetup = suite._isProjectSetup; + test._phase = suite._phase; suite._addTest(test); if (type === 'only' || type === 'projectSetupOnly') @@ -134,7 +134,7 @@ export class TestTypeImpl { const child = new Suite(title, 'describe'); child._requireFile = suite._requireFile; - child._isProjectSetup = suite._isProjectSetup; + child._phase = suite._phase; child.location = location; suite._addSuite(child); diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index a261ae2551..1a3e6fba95 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -54,6 +54,10 @@ export interface FullConfigInternal extends FullConfigPublic { */ webServer: FullConfigPublic['webServer']; _webServers: Exclude[]; + _globalScripts: string | RegExp | (string | RegExp)[] | null; + + // This is an ephemeral project that is not added to `projects` list below. + _globalProject: FullProjectInternal; // Overrides the public field. projects: FullProjectInternal[]; diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index e50e832ba9..9793776858 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -160,7 +160,11 @@ export class WorkerRunner extends EventEmitter { return; this._loader = await Loader.deserialize(this._params.loader); - this._project = this._loader.fullConfig().projects.find(p => p._id === this._params.projectId)!; + const globalProject = this._loader.fullConfig()._globalProject; + if (this._params.projectId === globalProject._id) + this._project = globalProject; + else + this._project = this._loader.fullConfig().projects.find(p => p._id === this._params.projectId)!; } async runTestGroup(runPayload: RunPayload) { @@ -169,7 +173,7 @@ export class WorkerRunner extends EventEmitter { let fatalUnknownTestIds; try { await this._loadIfNeeded(); - const fileSuite = await this._loader.loadTestFile(runPayload.file, 'worker', runPayload.projectSetup); + const fileSuite = await this._loader.loadTestFile(runPayload.file, 'worker', runPayload.phase); const suite = this._loader.buildFileSuiteForProject(this._project, fileSuite, this._params.repeatEachIndex, test => { if (runPayload.watchMode) { const testResolvedPayload: WatchTestResolvedPayload = { diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index ff8dec2c46..0223aee199 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -646,6 +646,22 @@ interface TestConfig { */ fullyParallel?: boolean; + /** + * Files that contain global setup/teardown hooks. + * + * **Details** + * + * [test.beforeAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-before-all) hooks in the matching + * files will run before testing starts. + * [test.afterAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-all) hooks in the matching + * files will run after testing finishes. + * + * If global setup fails, test execution will be skipped. + * [test.afterAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-after-all) hooks will run in the same + * worker process as [test.beforeAll(hookFunction)](https://playwright.dev/docs/api/class-test#test-before-all). + */ + globalScripts?: string|RegExp|Array; + /** * Path to the global setup file. This file will be required and run before all the tests. It must export a single * function that takes a [`TestConfig`] argument. diff --git a/tests/playwright-test/global-scripts.spec.ts b/tests/playwright-test/global-scripts.spec.ts new file mode 100644 index 0000000000..6ca46b0926 --- /dev/null +++ b/tests/playwright-test/global-scripts.spec.ts @@ -0,0 +1,225 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import path from 'path'; +import { test, expect } from './playwright-test-fixtures'; + +type Timeline = { titlePath: string[], event: 'begin' | 'end' }[]; + +function formatTimeline(timeline: Timeline) { + return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n'); +} + +function formatFileNames(timeline: Timeline) { + return timeline.map(e => e.titlePath[2]).join('\n'); +} + +function fileNames(timeline: Timeline) { + const fileNames = Array.from(new Set(timeline.map(({ titlePath }) => { + const name = titlePath[2]; + const index = name.lastIndexOf(path.sep); + if (index === -1) + return name; + return name.slice(index + 1); + })).keys()); + fileNames.sort(); + return fileNames; +} + +function expectFilesRunBefore(timeline: Timeline, before: string[], after: string[]) { + const fileBegin = name => { + const index = timeline.findIndex(({ titlePath }) => titlePath[2] === name); + expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1); + return index; + }; + const fileEnd = name => { + // There is no Array.findLastIndex in Node < 18. + let index = -1; + for (index = timeline.length - 1; index >= 0; index--) { + if (timeline[index].titlePath[2] === name) + break; + } + expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1); + return index; + }; + + for (const b of before) { + const bEnd = fileEnd(b); + for (const a of after) { + const aBegin = fileBegin(a); + expect(bEnd < aBegin, `'${b}' expected to finish before ${a}, actual order:\n${formatTimeline(timeline)}`).toBeTruthy(); + } + } +} + +test('should work for one project', async ({ runGroups }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + globalScripts: /.*global.ts/, + projects: [ + { + name: 'p1', + testMatch: /.*.test.ts/, + }, + ] + };`, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'global.ts': ` + const { test } = pwt; + test('setup1', async () => { }); + test('setup2', async () => { }); + `, + }; + const { exitCode, passed, timeline } = await runGroups(files); + expect(exitCode).toBe(0); + expect(passed).toBe(4); + expect(formatTimeline(timeline)).toEqual(`Global Scripts > global.ts > setup1 [begin] +Global Scripts > global.ts > setup1 [end] +Global Scripts > global.ts > setup2 [begin] +Global Scripts > global.ts > setup2 [end] +p1 > a.test.ts > test1 [begin] +p1 > a.test.ts > test1 [end] +p1 > a.test.ts > test2 [begin] +p1 > a.test.ts > test2 [end]`); +}); + +test('should work for several projects', async ({ runGroups }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + globalScripts: /.*global.ts/, + projects: [ + { + name: 'p1', + testMatch: /.*a.test.ts/, + }, + { + name: 'p2', + testMatch: /.*b.test.ts/, + }, + ] + };`, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'b.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'global.ts': ` + const { test } = pwt; + test('setup1', async () => { }); + test('setup2', async () => { }); + `, + }; + const { exitCode, passed, timeline } = await runGroups(files); + expect(exitCode).toBe(0); + expect(passed).toBe(6); + expectFilesRunBefore(timeline, [`global.ts`], [`a.test.ts`, `b.test.ts`]); +}); + +test('should skip tests if global setup fails', async ({ runGroups }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + globalScripts: /.*global.ts/, + projects: [ + { + name: 'p1', + testMatch: /.*a.test.ts/, + }, + { + name: 'p2', + testMatch: /.*b.test.ts/, + }, + ] + };`, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'b.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + `, + 'global.ts': ` + const { test, expect } = pwt; + test('setup1', async () => { }); + test('setup2', async () => { expect(1).toBe(2) }); + `, + }; + const { exitCode, passed, skipped } = await runGroups(files); + expect(exitCode).toBe(1); + expect(passed).toBe(1); + expect(skipped).toBe(3); +}); + +test('should run setup in each project shard', async ({ runGroups }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + globalScripts: /.*global.ts/, + projects: [ + { + name: 'p1', + }, + ] + };`, + 'a.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + test('test3', async () => { }); + test('test4', async () => { }); + `, + 'b.test.ts': ` + const { test } = pwt; + test('test1', async () => { }); + test('test2', async () => { }); + `, + 'global.ts': ` + const { test, expect } = pwt; + test('setup1', async () => { }); + test('setup2', async () => { }); + `, + }; + + { // Shard 1/2 + const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '1/2' }); + expect(output).toContain('Running 6 tests using 1 worker, shard 1 of 2'); + expect(fileNames(timeline)).toEqual(['a.test.ts', 'global.ts']); + expectFilesRunBefore(timeline, [`global.ts`], [`a.test.ts`]); + expect(exitCode).toBe(0); + expect(passed).toBe(6); + } + { // Shard 2/2 + const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '2/2' }); + expect(output).toContain('Running 4 tests using 1 worker, shard 2 of 2'); + expect(fileNames(timeline)).toEqual(['b.test.ts', 'global.ts']); + expectFilesRunBefore(timeline, [`global.ts`], [`b.test.ts`]); + expect(exitCode).toBe(0); + expect(passed).toBe(4); + } +}); + diff --git a/tests/playwright-test/project-setup.spec.ts b/tests/playwright-test/project-setup.spec.ts index 8b747219e3..4166c634cc 100644 --- a/tests/playwright-test/project-setup.spec.ts +++ b/tests/playwright-test/project-setup.spec.ts @@ -892,7 +892,7 @@ test('should prohibit beforeAll hooks in setup files', async ({ runGroups }, tes const { exitCode, output } = await runGroups(files); expect(exitCode).toBe(1); - expect(output).toContain('test.beforeAll() is not allowed in a project setup file'); + expect(output).toContain('test.beforeAll() is not allowed in a setup file'); }); test('should prohibit test in setup files', async ({ runGroups }, testInfo) => { @@ -914,7 +914,7 @@ test('should prohibit test in setup files', async ({ runGroups }, testInfo) => { const { exitCode, output } = await runGroups(files); expect(exitCode).toBe(1); - expect(output).toContain('test() is not allowed in a project setup file'); + expect(output).toContain('test() is not allowed in a setup file'); }); test('should prohibit test hooks in setup files', async ({ runGroups }, testInfo) => { @@ -936,5 +936,5 @@ test('should prohibit test hooks in setup files', async ({ runGroups }, testInfo const { exitCode, output } = await runGroups(files); expect(exitCode).toBe(1); - expect(output).toContain('test.beforeEach() is not allowed in a project setup file'); + expect(output).toContain('test.beforeEach() is not allowed in a setup file'); });