From 10d7c60abf172b7ae23a4939bdc30279044fd35e Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 23 Sep 2022 20:01:27 -0700 Subject: [PATCH] feat(runner): project execution schedule (#17548) --- docs/src/test-api/class-testconfig.md | 7 + packages/playwright-test/src/cli.ts | 2 + packages/playwright-test/src/loader.ts | 55 +++- .../playwright-test/src/reporters/base.ts | 2 +- packages/playwright-test/src/runner.ts | 253 +++++++++++------ packages/playwright-test/src/types.ts | 10 +- packages/playwright-test/types/test.d.ts | 10 + tests/playwright-test/config.spec.ts | 48 ++++ tests/playwright-test/groups.spec.ts | 260 ++++++++++++++++++ .../playwright-test-fixtures.ts | 40 ++- 10 files changed, 596 insertions(+), 91 deletions(-) create mode 100644 tests/playwright-test/groups.spec.ts diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 33f2c91bdf..0bb4a857c5 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -232,6 +232,13 @@ Filter to only run tests with a title **not** matching one of the patterns. This `grepInvert` option is also useful for [tagging tests](../test-annotations.md#tag-tests). +## property: TestConfig.groups +* since: v1.27 +- type: ?<[Object]<[string],[Array]<[string]|[Array]<[string]|[Object]>>>> + - `project` <[string]|[Array]<[string]>> Project name(s). + +Project groups that control project execution order. + ## property: TestConfig.ignoreSnapshots * since: v1.26 - type: ?<[boolean]> diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index ca44de2a78..cbadeabbce 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -58,6 +58,7 @@ function addTestCommand(program: Command) { command.option('--retries ', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`); command.option('--shard ', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`); command.option('--project ', `Only run tests from the specified list of projects (default: run all projects)`); + command.option('--group ', `Only run tests from the specified project group (default: run all projects from the 'default' group or just all projects if 'default' group is not defined).`); command.option('--timeout ', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`); command.option('--trace ', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`); command.option('-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`); @@ -166,6 +167,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) { listOnly: !!opts.list, testFileFilters, projectFilter: opts.project || undefined, + projectGroup: opts.group, watchMode: !!process.env.PW_TEST_WATCH, passWithNoTests: opts.passWithNoTests, }); diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index a3ab9bc222..d15623ceb5 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -27,7 +27,7 @@ import * as os from 'os'; import type { BuiltInReporter, ConfigCLIOverrides } from './runner'; import type { Reporter } from '../types/testReporter'; import { builtInReporters } from './runner'; -import { isRegExp, calculateSha1 } from 'playwright-core/lib/utils'; +import { isRegExp, calculateSha1, isString, isObject } from 'playwright-core/lib/utils'; import { serializeError } from './util'; import { FixturePool, isFixtureOption } from './fixtures'; import type { TestTypeImpl } from './testType'; @@ -167,6 +167,7 @@ export class Loader { this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata); this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath)); this._assignUniqueProjectIds(this._fullConfig.projects); + this._fullConfig.groups = config.groups; } private _assignUniqueProjectIds(projects: FullProjectInternal[]) { @@ -538,6 +539,8 @@ function validateConfig(file: string, config: Config) { }); } + validateProjectGroups(file, config); + if ('quiet' in config && config.quiet !== undefined) { if (typeof config.quiet !== 'boolean') throw errorWithFile(file, `config.quiet must be a boolean`); @@ -644,6 +647,54 @@ function validateProject(file: string, project: Project, title: string) { } } +function validateProjectGroups(file: string, config: Config) { + if (config.groups === undefined) + return; + const projectNames = new Set(config.projects?.filter(p => !!p.name).map(p => p.name)); + for (const [groupName, group] of Object.entries(config.groups)) { + function validateProjectReference(projectName: string) { + if (projectName.trim() === '') + throw errorWithFile(file, `config.groups.${groupName} refers to an empty project name`); + if (!projectNames.has(projectName)) + throw errorWithFile(file, `config.groups.${groupName} refers to an unknown project '${projectName}'`); + } + for (const step of group) { + if (isString(step)) { + validateProjectReference(step); + } else if (Array.isArray(step)) { + const parallelProjectNames = new Set(); + for (const item of step) { + let projectName; + if (isString(item)) { + validateProjectReference(item); + projectName = item; + } else if (isObject(item)) { + const project = (item as any).project; + if (isString(project)) { + validateProjectReference(project); + } else if (Array.isArray(project)) { + project.forEach(name => { + if (!isString(name)) + throw errorWithFile(file, `config.groups.${groupName}[*].project contains non string value.`); + validateProjectReference(name); + }); + } + projectName = project; + } else { + throw errorWithFile(file, `config.groups.${groupName} unexpected group entry ${JSON.stringify(step, null, 2)}`); + } + // We can relax this later. + if (parallelProjectNames.has(projectName)) + throw errorWithFile(file, `config.groups.${groupName} group mentions project '${projectName}' twice in one parallel group`); + parallelProjectNames.add(projectName); + } + } else { + throw errorWithFile(file, `config.groups.${groupName} unexpected group entry ${JSON.stringify(step, null, 2)}`); + } + } + } +} + export const baseFullConfig: FullConfigInternal = { forbidOnly: false, fullyParallel: false, @@ -670,7 +721,7 @@ export const baseFullConfig: FullConfigInternal = { _webServers: [], _globalOutputDir: path.resolve(process.cwd()), _configDir: '', - _testGroupsCount: 0, + _maxConcurrentTestGroups: 0, _ignoreSnapshots: false, _workerIsolation: 'isolate-pools', }; diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index cb234a2220..2a4a653560 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -121,7 +121,7 @@ export class BaseReporter implements ReporterInternal { } protected generateStartingMessage() { - const jobs = Math.min(this.config.workers, this.config._testGroupsCount); + const jobs = Math.min(this.config.workers, this.config._maxConcurrentTestGroups); const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; if (this.config._watchMode) return `\nRunning tests in the --watch mode`; diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 6ea084162f..528bc4c0d8 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -47,16 +47,24 @@ import { setRunnerToAddPluginsTo } from './plugins'; import { webServerPluginsForConfig } from './plugins/webServerPlugin'; import { dockerPlugin } from './plugins/dockerPlugin'; import { MultiMap } from 'playwright-core/lib/utils/multimap'; +import { isString, assert } from 'playwright-core/lib/utils'; const removeFolderAsync = promisify(rimraf); const readDirAsync = promisify(fs.readdir); const readFileAsync = promisify(fs.readFile); export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs']; +// Project group is a sequence of run phases. +type RunPhase = { + testFileFilters: TestFileFilter[]; + projectFilter?: string[]; +}; + type RunOptions = { listOnly?: boolean; testFileFilters?: TestFileFilter[]; projectFilter?: string[]; + projectGroup?: string; watchMode?: boolean; passWithNoTests?: boolean; }; @@ -227,6 +235,58 @@ export class Runner { return report; } + + private _collectRunPhases(options: RunOptions) { + const config = this._loader.fullConfig(); + + let projectGroup = options.projectGroup; + if (options.projectFilter) { + if (projectGroup) + throw new Error('--group option can not be combined with --project'); + } else { + if (!projectGroup && config.groups?.default && !options.testFileFilters?.length) + projectGroup = 'default'; + if (projectGroup) { + if (config.shard) + throw new Error(`Project group '${projectGroup}' cannot be combined with --shard`); + } + } + + const phases: RunPhase[] = []; + if (projectGroup) { + const group = config.groups?.[projectGroup]; + if (!group) + throw new Error(`Cannot find project group '${projectGroup}' in the config`); + for (const entry of group) { + const projectFilter: string[] = []; + const testFileFilters: TestFileFilter[] = []; + if (isString(entry)) { + projectFilter.push(entry); + } else { + for (const p of entry) { + if (isString(p)) + projectFilter.push(p); + else if (isString(p.project)) + projectFilter.push(p.project); + else + projectFilter.push(...p.project); + } + } + // TODO: filter per project set. + phases.push({ + testFileFilters, + projectFilter + }); + } + } else { + phases.push({ + projectFilter: options.projectFilter, + testFileFilters: options.testFileFilters || [], + }); + } + return phases; + } + private async _collectFiles(testFileFilters: TestFileFilter[], projectNames?: string[]): Promise> { const testFileFilter = testFileFilters.length ? createFileMatcher(testFileFilters.map(e => e.re || e.exact || '')) : () => true; let projectsToFind: Set | undefined; @@ -270,74 +330,86 @@ export class Runner { } private async _run(options: RunOptions): Promise { - const testFileFilters = options.testFileFilters || []; - const filesByProject = await this._collectFiles(testFileFilters, options.projectFilter); - - const allTestFiles = new Set(); - for (const files of filesByProject.values()) - files.forEach(file => allTestFiles.add(file)); - const config = this._loader.fullConfig(); - const fatalErrors: TestError[] = []; - - // 1. Add all tests. - const preprocessRoot = new Suite('', 'root'); - for (const file of allTestFiles) { - const fileSuite = await this._loader.loadTestFile(file, 'runner'); - if (fileSuite._loadError) - fatalErrors.push(fileSuite._loadError); - preprocessRoot._addSuite(fileSuite); - } - - // 2. Complain about duplicate titles. - const duplicateTitlesError = createDuplicateTitlesError(config, preprocessRoot); - if (duplicateTitlesError) - fatalErrors.push(duplicateTitlesError); - - // 3. Filter tests to respect line/column filter. - filterByFocusedLine(preprocessRoot, testFileFilters); - - // 4. Complain about only. - if (config.forbidOnly) { - const onlyTestsAndSuites = preprocessRoot._getOnlyItems(); - if (onlyTestsAndSuites.length > 0) - fatalErrors.push(createForbidOnlyError(config, onlyTestsAndSuites)); - } - - // 5. Filter only. - if (!options.listOnly) - filterOnly(preprocessRoot); - - // 6. Generate projects. - const fileSuites = new Map(); - for (const fileSuite of preprocessRoot.suites) - fileSuites.set(fileSuite._requireFile, fileSuite); - + // Each entry is an array of test groups that can be run concurrently. All + // test groups from the previos entries must finish before entry starts. + const concurrentTestGroups = []; const rootSuite = new Suite('', 'root'); - for (const [project, files] of filesByProject) { - const grepMatcher = createTitleMatcher(project.grep); - const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null; - const projectSuite = new Suite(project.name, 'project'); - projectSuite._projectConfig = project; - if (project._fullyParallel) - projectSuite._parallelMode = 'parallel'; - rootSuite._addSuite(projectSuite); - for (const file of files) { - const fileSuite = fileSuites.get(file); - if (!fileSuite) - continue; - for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) { - const builtSuite = this._loader.buildFileSuiteForProject(project, fileSuite, repeatEachIndex, test => { - const grepTitle = test.titlePath().join(' '); - if (grepInvertMatcher?.(grepTitle)) - return false; - return grepMatcher(grepTitle); - }); - if (builtSuite) - projectSuite._addSuite(builtSuite); + const runPhases = this._collectRunPhases(options); + assert(runPhases.length > 0); + for (const { projectFilter, testFileFilters } of runPhases) { + // TODO: do not collect files for each project multiple times. + const filesByProject = await this._collectFiles(testFileFilters, projectFilter); + + const allTestFiles = new Set(); + for (const files of filesByProject.values()) + files.forEach(file => allTestFiles.add(file)); + + + // 1. Add all tests. + const preprocessRoot = new Suite('', 'root'); + for (const file of allTestFiles) { + const fileSuite = await this._loader.loadTestFile(file, 'runner'); + if (fileSuite._loadError) + fatalErrors.push(fileSuite._loadError); + preprocessRoot._addSuite(fileSuite); + } + + // 2. Complain about duplicate titles. + const duplicateTitlesError = createDuplicateTitlesError(config, preprocessRoot); + if (duplicateTitlesError) + fatalErrors.push(duplicateTitlesError); + + // 3. Filter tests to respect line/column filter. + // TODO: figure out how this is supposed to work with groups. + filterByFocusedLine(preprocessRoot, testFileFilters); + + // 4. Complain about only. + if (config.forbidOnly) { + const onlyTestsAndSuites = preprocessRoot._getOnlyItems(); + if (onlyTestsAndSuites.length > 0) + fatalErrors.push(createForbidOnlyError(config, onlyTestsAndSuites)); + } + + // 5. Filter only. + if (!options.listOnly) + filterOnly(preprocessRoot); + + // 6. Generate projects. + const fileSuites = new Map(); + for (const fileSuite of preprocessRoot.suites) + fileSuites.set(fileSuite._requireFile, fileSuite); + + const firstProjectSuiteIndex = rootSuite.suites.length; + for (const [project, files] of filesByProject) { + const grepMatcher = createTitleMatcher(project.grep); + const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null; + const projectSuite = new Suite(project.name, 'project'); + projectSuite._projectConfig = project; + if (project._fullyParallel) + projectSuite._parallelMode = 'parallel'; + rootSuite._addSuite(projectSuite); + for (const file of files) { + const fileSuite = fileSuites.get(file); + if (!fileSuite) + continue; + for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) { + const builtSuite = this._loader.buildFileSuiteForProject(project, fileSuite, repeatEachIndex, test => { + const grepTitle = test.titlePath().join(' '); + if (grepInvertMatcher?.(grepTitle)) + return false; + return grepMatcher(grepTitle); + }); + if (builtSuite) + projectSuite._addSuite(builtSuite); + } } } + + const projectSuites = rootSuite.suites.slice(firstProjectSuiteIndex); + const testGroups = createTestGroups(projectSuites, config.workers); + concurrentTestGroups.push(testGroups); } // 7. Fail when no tests. @@ -346,10 +418,10 @@ export class Runner { fatalErrors.push(createNoTestsError()); // 8. Compute shards. - let testGroups = createTestGroups(rootSuite, config.workers); - const shard = config.shard; if (shard) { + assert(!options.projectGroup); + assert(concurrentTestGroups.length === 1); const shardGroups: TestGroup[] = []; const shardTests = new Set(); @@ -362,7 +434,7 @@ export class Runner { const from = shardSize * currentShard + Math.min(extraOne, currentShard); const to = from + shardSize + (currentShard < extraOne ? 1 : 0); let current = 0; - for (const group of testGroups) { + for (const group of concurrentTestGroups[0]) { // Any test group goes to the shard that contains the first test of this group. // So, this shard gets any group that starts at [from; to) if (current >= from && current < to) { @@ -373,11 +445,12 @@ export class Runner { current += group.tests.length; } - testGroups = shardGroups; + concurrentTestGroups[0] = shardGroups; filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test)); total = rootSuite.allTests().length; } - config._testGroupsCount = testGroups.length; + + config._maxConcurrentTestGroups = Math.max(...concurrentTestGroups.map(g => g.length)); // 9. Report begin this._reporter.onBegin?.(config, rootSuite); @@ -399,7 +472,7 @@ export class Runner { // 13. Run Global setup. const result: FullResult = { status: 'passed' }; - const globalTearDown = await this._performGlobalSetup(config, rootSuite, [...filesByProject.keys()], result); + const globalTearDown = await this._performGlobalSetup(config, rootSuite, result); if (result.status !== 'passed') return result; @@ -414,24 +487,32 @@ export class Runner { // 14. Run tests. try { - const sigintWatcher = new SigIntWatcher(); + let sigintWatcher; let hasWorkerErrors = false; - const dispatcher = new Dispatcher(this._loader, testGroups, this._reporter); - await Promise.race([dispatcher.run(), sigintWatcher.promise()]); - if (!sigintWatcher.hadSignal()) { - // We know for sure there was no Ctrl+C, so we remove custom SIGINT handler - // as soon as we can. - sigintWatcher.disarm(); + for (const testGroups of concurrentTestGroups) { + const dispatcher = new Dispatcher(this._loader, testGroups, this._reporter); + sigintWatcher = new SigIntWatcher(); + await Promise.race([dispatcher.run(), sigintWatcher.promise()]); + if (!sigintWatcher.hadSignal()) { + // We know for sure there was no Ctrl+C, so we remove custom SIGINT handler + // as soon as we can. + sigintWatcher.disarm(); + } + await dispatcher.stop(); + hasWorkerErrors = dispatcher.hasWorkerErrors(); + if (hasWorkerErrors) + break; + if (testGroups.some(testGroup => testGroup.tests.some(test => !test.ok()))) + break; + if (sigintWatcher.hadSignal()) + break; } - await dispatcher.stop(); - hasWorkerErrors = dispatcher.hasWorkerErrors(); - - if (!sigintWatcher.hadSignal()) { + if (sigintWatcher?.hadSignal()) { + result.status = 'interrupted'; + } else { const failed = hasWorkerErrors || rootSuite.allTests().some(test => !test.ok()); result.status = failed ? 'failed' : 'passed'; - } else { - result.status = 'interrupted'; } } catch (e) { this._reporter.onError?.(serializeError(e)); @@ -457,7 +538,7 @@ export class Runner { // 4. Run Global setup. const result: FullResult = { status: 'passed' }; - const globalTearDown = await this._performGlobalSetup(config, rootSuite, config.projects.filter(p => !options.projectFilter || options.projectFilter.includes(p.name)), result); + const globalTearDown = await this._performGlobalSetup(config, rootSuite, result); if (result.status !== 'passed') return result; @@ -590,7 +671,7 @@ export class Runner { return true; } - private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, projects: FullProjectInternal[], result: FullResult): Promise<(() => Promise) | undefined> { + private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, result: FullResult): Promise<(() => Promise) | undefined> { let globalSetupResult: any = undefined; const pluginsThatWereSetUp: TestRunnerPlugin[] = []; @@ -805,7 +886,7 @@ function buildItemLocation(rootDir: string, testOrSuite: Suite | TestCase) { return `${path.relative(rootDir, testOrSuite.location.file)}:${testOrSuite.location.line}`; } -function createTestGroups(rootSuite: Suite, workers: number): TestGroup[] { +function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[] { // This function groups tests that can be run together. // Tests cannot be run together when: // - They belong to different projects - requires different workers. @@ -843,7 +924,7 @@ function createTestGroups(rootSuite: Suite, workers: number): TestGroup[] { }; }; - for (const projectSuite of rootSuite.suites) { + for (const projectSuite of projectSuites) { for (const test of projectSuite.allTests()) { let withWorkerHash = groups.get(test._workerHash); if (!withWorkerHash) { diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index 8bcac6c47d..5dce705604 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -44,7 +44,7 @@ export interface TestStepInternal { export interface FullConfigInternal extends FullConfigPublic { _globalOutputDir: string; _configDir: string; - _testGroupsCount: number; + _maxConcurrentTestGroups: number; _watchMode: boolean; _ignoreSnapshots: boolean; _workerIsolation: WorkerIsolation; @@ -56,6 +56,14 @@ export interface FullConfigInternal extends FullConfigPublic { // Overrides the public field. projects: FullProjectInternal[]; + + groups?: { [key: string]: Array, + testIgnore?: string | RegExp | Array + }>> }; } /** diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index de43cab5c7..47f073679d 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -687,6 +687,16 @@ interface TestConfig { */ grepInvert?: RegExp|Array; + /** + * Project groups that control project execution order. + */ + groups?: { [key: string]: Array; + }>>; }; + /** * Whether to skip snapshot expectations, such as `expect(value).toMatchSnapshot()` and `await * expect(page).toHaveScreenshot()`. diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index 23082e0a60..76ddf2d3e1 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -480,3 +480,51 @@ test('should have correct types for the config', async ({ runTSC }) => { }); expect(result.exitCode).toBe(0); }); + +test('should throw when group has duplicate project references', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'a' }, + ], + groups: { + default: [ + ['a', 'a'] + ] + } + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async () => {}); + ` + }); + + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`config.groups.default group mentions project 'a' twice in one parallel group`); +}); + +test('should throw when group has unknown project reference', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'a' }, + ], + groups: { + default: [ + [{project: 'b'}] + ] + } + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async () => {}); + ` + }); + + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`config.groups.default refers to an unknown project 'b'`); +}); diff --git a/tests/playwright-test/groups.spec.ts b/tests/playwright-test/groups.spec.ts new file mode 100644 index 0000000000..c0ad3f55d8 --- /dev/null +++ b/tests/playwright-test/groups.spec.ts @@ -0,0 +1,260 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import type { PlaywrightTestConfig, TestInfo } from '@playwright/test'; +import path from 'path'; +import { test, expect } from './playwright-test-fixtures'; + +function createConfigWithProjects(names: string[], testInfo: TestInfo, groups: PlaywrightTestConfig['groups']): Record { + const config: PlaywrightTestConfig = { + projects: names.map(name => ({ name, testDir: testInfo.outputPath(name) })), + groups + }; + const files = {}; + for (const name of names) { + files[`${name}/${name}.spec.ts`] = ` + const { test } = pwt; + test('${name} test', async () => { + await new Promise(f => setTimeout(f, 100)); + });`; + } + files['playwright.config.ts'] = ` + import * as path from 'path'; + module.exports = ${JSON.stringify(config)}; + `; + return files; +} + +type Timeline = { titlePath: string[], event: 'begin' | 'end' }[]; + +function formatTimeline(timeline: Timeline) { + return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n'); +} + +function expectRunBefore(timeline: Timeline, before: string[], after: string[]) { + const begin = new Map(); + const end = new Map(); + for (let i = 0; i < timeline.length; i++) { + const projectName = timeline[i].titlePath[1]; + const map = timeline[i].event === 'begin' ? begin : end; + const oldIndex = map.get(projectName) ?? i; + const newIndex = (timeline[i].event === 'begin') ? Math.min(i, oldIndex) : Math.max(i, oldIndex); + map.set(projectName, newIndex); + } + for (const b of before) { + for (const a of after) { + const bEnd = end.get(b) as number; + expect(bEnd === undefined, `Unknown project ${b}`).toBeFalsy(); + const aBegin = begin.get(a) as number; + expect(aBegin === undefined, `Unknown project ${a}`).toBeFalsy(); + if (bEnd < aBegin) + continue; + throw new Error(`Project '${b}' expected to finish before '${a}'\nTest run order was:\n${formatTimeline(timeline)}`); + } + } +} + +test('should work', async ({ runGroups }, testInfo) => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: ['a'] + }); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expect(passed).toBe(1); + expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin] +a > a${path.sep}a.spec.ts > a test [end]`); +}); + +test('should order two projects', async ({ runGroups }, testInfo) => { + await test.step(`order a then b`, async () => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: [ + 'a', + 'b' + ] + }); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expect(passed).toBe(2); + expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin] +a > a${path.sep}a.spec.ts > a test [end] +b > b${path.sep}b.spec.ts > b test [begin] +b > b${path.sep}b.spec.ts > b test [end]`); + }); + await test.step(`order b then a`, async () => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: [ + 'b', + 'a' + ] + }); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expect(passed).toBe(2); + expect(formatTimeline(timeline)).toEqual(`b > b${path.sep}b.spec.ts > b test [begin] +b > b${path.sep}b.spec.ts > b test [end] +a > a${path.sep}a.spec.ts > a test [begin] +a > a${path.sep}a.spec.ts > a test [end]`); + }); +}); + +test('should order 1-3-1 projects', async ({ runGroups }, testInfo) => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: [ + 'e', + ['d', 'c', 'b'], + 'a', + ] + }); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expectRunBefore(timeline, ['e'], ['d', 'c', 'b']); + expectRunBefore(timeline, ['d', 'c', 'b'], ['a']); + expect(passed).toBe(5); +}); + +test('should order 2-2-2 projects', async ({ runGroups }, testInfo) => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: [ + ['a', 'b'], + ['d', 'c'], + ['e', 'f'], + ] + }); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expectRunBefore(timeline, ['a', 'b'], ['c', 'd']); + expectRunBefore(timeline, ['c', 'd'], ['e', 'f']); + expect(passed).toBe(6); +}); + +test('should run parallel groups sequentially without overlaps', async ({ runGroups }, testInfo) => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: [ + ['a', 'b', 'c', 'd'], + ['a', 'b', 'c', 'd'], + ['a', 'b', 'c', 'd'], + ] + }); + const { exitCode, passed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + + const expectedEndOfFirstPhase = events => { + const firstProjectEndIndex = project => events.findIndex(e => e.event === 'end' && e.titlePath[1] === project); + return Math.max(...['a', 'b', 'c', 'd'].map(firstProjectEndIndex)); + }; + const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event); + + let remainingTimeline = timeline; + for (let i = 0; i < 3; i++) { + const phaseEndIndex = expectedEndOfFirstPhase(remainingTimeline); + const firstPhase = formatPhaseEvents(remainingTimeline.slice(0, phaseEndIndex + 1)); + firstPhase.sort(); + expect(firstPhase, `check phase ${i}`).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end', 'c:begin', 'c:end', 'd:begin', 'd:end']); + remainingTimeline = remainingTimeline.slice(phaseEndIndex + 1); + } + expect(remainingTimeline.length).toBe(0); + + expect(passed).toBe(12); +}); + +test('should support phase with multiple project names', async ({ runGroups }, testInfo) => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: [ + [ + { project: ['a', 'b', 'c'] } + ], + [ + { project: ['d'] }, + { project: ['e', 'f'] } + ], + ] + }); + + const { exitCode, passed } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expect(passed).toBe(6); +}); + +test('should support varios syntax', async ({ runGroups }, testInfo) => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: [ + 'a', + ['a', 'b'], + [ + { project: ['a', 'b'] } + ], + [ + { project: ['a', 'b'] }, + 'c', + { project: 'd' }, + ], + [{ project: 'e' }], + 'f' + ] + }); + const { exitCode, passed } = await runGroups(configWithFiles); + expect(exitCode).toBe(0); + expect(passed).toBe(11); +}); + +test('should support --group option', async ({ runGroups }, testInfo) => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: [ + 'a', 'b' + ], + foo: [ + ['b', 'c'] + ], + bar: [ + 'd', 'e' + ] + }); + const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event); + { + const { exitCode, passed, timeline } = await runGroups(configWithFiles, { group: 'default' }); + expect(exitCode).toBe(0); + expect(passed).toBe(2); + expect(formatPhaseEvents(timeline)).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end']); + } + { + const { exitCode, passed, timeline } = await runGroups(configWithFiles, { group: 'foo' }); + expect(exitCode).toBe(0); + expect(passed).toBe(2); + const formatted = formatPhaseEvents(timeline); + formatted.sort(); + expect(formatted).toEqual(['b:begin', 'b:end', 'c:begin', 'c:end']); + } + { + const { exitCode, passed, timeline } = await runGroups(configWithFiles, { group: 'bar' }); + expect(exitCode).toBe(0); + expect(passed).toBe(2); + expect(formatPhaseEvents(timeline)).toEqual(['d:begin', 'd:end', 'e:begin', 'e:end']); + } +}); + +test('should throw when unknown --group is passed', async ({ runGroups }, testInfo) => { + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, { + default: [ + 'a', 'b' + ], + foo: [ + ['b', 'c'] + ] + }); + const { exitCode, output } = await runGroups(configWithFiles, { group: 'bar' }); + expect(exitCode).toBe(1); + expect(output).toContain(`Cannot find project group 'bar' in the config`); +}); diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index fd3e36e548..78100dcdf8 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -224,7 +224,8 @@ type Fixtures = { writeFiles: (files: Files) => Promise; runInlineTest: (files: Files, params?: Params, env?: Env, options?: RunOptions, beforeRunPlaywrightTest?: ({ baseDir }: { baseDir: string }) => Promise) => Promise; runTSC: (files: Files) => Promise; - nodeVersion: { major: number, minor: number, patch: number }, + nodeVersion: { major: number, minor: number, patch: number }; + runGroups: (files: Files, params?: Params, env?: Env, options?: RunOptions) => Promise<{ timeline: { titlePath: string[], event: 'begin' | 'end' }[] } & RunResult>; }; export const test = base @@ -261,6 +262,43 @@ export const test = base const [major, minor, patch] = process.versions.node.split('.'); await use({ major: +major, minor: +minor, patch: +patch }); }, + + runGroups: async ({ runInlineTest }, use, testInfo) => { + const timelinePath = testInfo.outputPath('timeline.json'); + await use(async (files, params, env, options) => { + const result = await runInlineTest({ + ...files, + 'reporter.ts': ` + import { Reporter, TestCase } from '@playwright/test/reporter'; + import fs from 'fs'; + import path from 'path'; + class TimelineReporter implements Reporter { + private _timeline: {titlePath: string, event: 'begin' | 'end'}[] = []; + onTestBegin(test: TestCase) { + this._timeline.push({ titlePath: test.titlePath(), event: 'begin' }); + } + onTestEnd(test: TestCase) { + this._timeline.push({ titlePath: test.titlePath(), event: 'end' }); + } + onEnd() { + fs.writeFileSync(path.join(${JSON.stringify(timelinePath)}), JSON.stringify(this._timeline, null, 2)); + } + } + export default TimelineReporter; + ` + }, { ...params, reporter: 'list,json,./reporter.ts', workers: 2 }, env, options); + + let timeline; + try { + timeline = JSON.parse((await fs.promises.readFile(timelinePath, 'utf8')).toString('utf8')); + } catch (e) { + } + return { + ...result, + timeline + }; + }); + }, }); const TSCONFIG = {