diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index fc21a4d9d9..e92a2f0313 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -482,6 +482,25 @@ export default defineConfig({ ``` +## property: TestConfig.shardingMode + +* since: v1.45 +- type: ?<[ShardingMode]<"partition"|"round-robin"|"duration-round-robin">> + +Defines the algorithm to be used for sharding. Defaults to `'partition'`. +* `'partition'` - divide the set of test groups by number of shards. e.g. first + half goes to shard 1/2 and seconds half to shard 2/2. +* `'round-robin'` - spread test groups to shards in a round-robin way. e.g. loop + over test groups and always assign to the shard that has the lowest number of + tests. +* `'duration-round-robin'` - use duration info from `.last-run.json` to spread + test groups to shards in a round-robin way. e.g. loop over test groups and + always assign to the shard that has the lowest duration of tests. new tests + which were not present in the last run will use an average duration time. When + no `.last-run.json` could be found the behavior is identical to + `'round-robin'`. + + ## property: TestConfig.shardingSeed * since: v1.45 diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 0caa6e32b5..f4b17b31e0 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import os from 'os'; -import type { Config, Fixtures, Project, ReporterDescription } from '../../types/test'; +import type { Config, Fixtures, PlaywrightTestConfig, Project, ReporterDescription } from '../../types/test'; import type { Location } from '../../types/testReporter'; import type { TestRunnerPluginRegistration } from '../plugins'; import { getPackageJsonPath, mergeObjects } from '../util'; @@ -25,6 +25,7 @@ import type { Matcher } from '../util'; import type { ConfigCLIOverrides } from './ipc'; import type { FullConfig, FullProject } from '../../types/testReporter'; import { setTransformConfig } from '../transform/transform'; +import type { LastRunInfo } from '../runner/runner'; export type ConfigLocation = { resolvedConfigFile?: string; @@ -55,7 +56,9 @@ export class FullConfigInternal { cliFailOnFlakyTests?: boolean; testIdMatcher?: Matcher; defineConfigWasUsed = false; + shardingMode: Exclude; shardingSeed: string | null; + lastRunInfo?: LastRunInfo; constructor(location: ConfigLocation, userConfig: Config, configCLIOverrides: ConfigCLIOverrides) { if (configCLIOverrides.projects && userConfig.projects) @@ -93,6 +96,7 @@ export class FullConfigInternal { workers: 0, webServer: null, }; + this.shardingMode = takeFirst(configCLIOverrides.shardingMode, userConfig.shardingMode, 'partition'); this.shardingSeed = takeFirst(configCLIOverrides.shardingSeed, userConfig.shardingSeed, null); for (const key in userConfig) { if (key.startsWith('@')) diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index f5e3ec0858..0fbed4b082 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -17,7 +17,7 @@ import util from 'util'; import { type SerializedCompilationCache, serializeCompilationCache } from '../transform/compilationCache'; import type { ConfigLocation, FullConfigInternal } from './config'; -import type { ReporterDescription, TestInfoError, TestStatus } from '../../types/test'; +import type { PlaywrightTestConfig, ReporterDescription, TestInfoError, TestStatus } from '../../types/test'; export type ConfigCLIOverrides = { forbidOnly?: boolean; @@ -32,6 +32,7 @@ export type ConfigCLIOverrides = { reporter?: ReporterDescription[]; additionalReporters?: ReporterDescription[]; shard?: { current: number, total: number }; + shardingMode?: PlaywrightTestConfig['shardingMode']; shardingSeed?: string; timeout?: number; ignoreSnapshots?: boolean; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index e85f3c4911..5a6f6212a9 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -184,9 +184,13 @@ async function runTests(args: string[], opts: { [key: string]: any }) { if (!config) return; - if (opts.lastFailed) { + if (opts.lastFailed || config.shardingMode === 'duration-round-robin') { const lastRunInfo = await readLastRunInfo(config); - config.testIdMatcher = id => lastRunInfo.failedTests.includes(id); + if (opts.lastFailed) + config.testIdMatcher = id => lastRunInfo.failedTests.includes(id); + + if (config.shardingMode === 'duration-round-robin') + config.lastRunInfo = lastRunInfo; } config.cliArgs = args; @@ -281,6 +285,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid retries: options.retries ? parseInt(options.retries, 10) : undefined, reporter: resolveReporterOption(options.reporter), shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined, + shardingMode: options.shardingMode ? options.shardingMode : undefined, shardingSeed: options.shardingSeed ? options.shardingSeed : undefined, timeout: options.timeout ? parseInt(options.timeout, 10) : undefined, ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined, @@ -359,6 +364,7 @@ const testOptions: [string, string][] = [ ['--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`], ['--retries ', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`], ['--shard ', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`], + ['--sharding-mode ', `Sharding algorithm to use; "partition", "round-robin" or "duration-round-robin". Defaults to "partition".`], ['--sharding-seed ', `Seed string for randomizing the test order before sharding. Defaults to not randomizing the order.`], ['--timeout ', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`], ['--trace ', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`], diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 304ee15dc9..f8a9d8554e 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -184,7 +184,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho shuffleWithSeed(testGroups, config.shardingSeed); // Shard test groups. - const testGroupsInThisShard = filterForShard(config.config.shard, testGroups); + const testGroupsInThisShard = filterForShard(config.shardingMode, config.config.shard, testGroups, config.lastRunInfo); const testsInThisShard = new Set(); for (const group of testGroupsInThisShard) { for (const test of group.tests) diff --git a/packages/playwright/src/runner/testGroups.ts b/packages/playwright/src/runner/testGroups.ts index 9b75cad399..01bb4a2d81 100644 --- a/packages/playwright/src/runner/testGroups.ts +++ b/packages/playwright/src/runner/testGroups.ts @@ -14,7 +14,9 @@ * limitations under the License. */ +import type { PlaywrightTestConfig } from '../../types/test'; import type { Suite, TestCase } from '../common/test'; +import type { LastRunInfo } from './runner'; export type TestGroup = { workerHash: string; @@ -130,7 +132,12 @@ export function createTestGroups(projectSuite: Suite, workers: number): TestGrou return result; } -export function filterForShard(shard: { total: number, current: number }, testGroups: TestGroup[]): Set { +export function filterForShard( + mode: PlaywrightTestConfig['shardingMode'], + shard: { total: number, current: number }, + testGroups: TestGroup[], + lastRunInfo?: LastRunInfo, +): Set { // Note that sharding works based on test groups. // This means parallel files will be sharded by single tests, // while non-parallel files will be sharded by the whole file. @@ -138,16 +145,85 @@ export function filterForShard(shard: { total: number, current: number }, testGr // Shards are still balanced by the number of tests, not files, // even in the case of non-paralleled files. - const lengths = new Array(shard.total).fill(0); + if (mode === 'round-robin') + return filterForShardRoundRobin(shard, testGroups); + if (mode === 'duration-round-robin') + return filterForShardRoundRobin(shard, testGroups, lastRunInfo); + return filterForShardPartition(shard, testGroups); +} + +/** + * Shards tests by partitioning them into equal parts. + * + * ``` + * [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + * Shard 1: ^---------^ : [ 1, 2, 3 ] + * Shard 2: ^---------^ : [ 4, 5, 6 ] + * Shard 3: ^---------^ : [ 7, 8, 9 ] + * Shard 4: ^---------^ : [ 10,11,12 ] + * ``` + */ +function filterForShardPartition(shard: { total: number, current: number }, testGroups: TestGroup[]): Set { + let shardableTotal = 0; + for (const group of testGroups) + shardableTotal += group.tests.length; + + // Each shard gets some tests. + const shardSize = Math.floor(shardableTotal / shard.total); + // First few shards get one more test each. + const extraOne = shardableTotal - shardSize * shard.total; + + const currentShard = shard.current - 1; // Make it zero-based for calculations. + const from = shardSize * currentShard + Math.min(extraOne, currentShard); + const to = from + shardSize + (currentShard < extraOne ? 1 : 0); + + let current = 0; + const result = new Set(); + for (const group of testGroups) { + // Any test group goes to the shard that contains the first test of this group. + // So, this shard gets any group that starts at [from; to) + if (current >= from && current < to) + result.add(group); + current += group.tests.length; + } + return result; +} + +/** + * Shards tests by round-robin. + * + * ``` + * [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + * Shard 1: ^ ^ ^ : [ 1, 5, 9 ] + * Shard 2: ^ ^ ^ : [ 2, 6,10 ] + * Shard 3: ^ ^ ^ : [ 3, 7,11 ] + * Shard 4: ^ ^ ^ : [ 4, 8,12 ] + * ``` + */ +function filterForShardRoundRobin( + shard: { total: number, current: number }, + testGroups: TestGroup[], + lastRunInfo?: LastRunInfo, +): Set { + + const weights = new Array(shard.total).fill(0); const shardSet = new Array(shard.total).fill(0).map(() => new Set()); + const averageDuration = lastRunInfo ? Object.values(lastRunInfo?.testDurations || {}).reduce((a, b) => a + b, 1) / Math.max(1, Object.values(lastRunInfo?.testDurations || {}).length) : 0; + const weight = (group: TestGroup) => { + if (!lastRunInfo) + // If we don't have last run info, we just count the number of tests. + return group.tests.length; + // If we have last run info, we use the duration of the tests. + return group.tests.reduce((sum, test) => sum + Math.max(1, lastRunInfo.testDurations?.[test.id] || averageDuration), 0); + }; // We sort the test groups by the number of tests in descending order. - const sortedTestGroups = testGroups.slice().sort((a, b) => b.tests.length - a.tests.length); + const sortedTestGroups = testGroups.slice().sort((a, b) => weight(b) - weight(a)); // Then we add each group to the shard with the smallest number of tests. for (const group of sortedTestGroups) { - const index = lengths.reduce((minIndex, currentLength, currentIndex) => currentLength < lengths[minIndex] ? currentIndex : minIndex, 0); - lengths[index] += group.tests.length; + const index = weights.reduce((minIndex, currentLength, currentIndex) => currentLength < weights[minIndex] ? currentIndex : minIndex, 0); + weights[index] += weight(group); shardSet[index].add(group); } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index e1299272a8..a1c05bbb34 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1425,6 +1425,19 @@ interface TestConfig { total: number; }; + /** + * Defines the algorithm to be used for sharding. Defaults to `'partition'`. + * - `'partition'` - divide the set of test groups by number of shards. e.g. first half goes to shard 1/2 and + * seconds half to shard 2/2. + * - `'round-robin'` - spread test groups to shards in a round-robin way. e.g. loop over test groups and always + * assign to the shard that has the lowest number of tests. + * - `'duration-round-robin'` - use duration info from `.last-run.json` to spread test groups to shards in a + * round-robin way. e.g. loop over test groups and always assign to the shard that has the lowest duration of + * tests. new tests which were not present in the last run will use an average duration time. When no + * `.last-run.json` could be found the behavior is identical to `'round-robin'`. + */ + shardingMode?: "partition"|"round-robin"|"duration-round-robin"; + /** * Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is * mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests diff --git a/tests/playwright-test/shard-roundrobin.spec.ts b/tests/playwright-test/shard-roundrobin.spec.ts new file mode 100644 index 0000000000..1aeecdcc0a --- /dev/null +++ b/tests/playwright-test/shard-roundrobin.spec.ts @@ -0,0 +1,457 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect, test } from './playwright-test-fixtures'; + +/** + * Test cases: + * - `a1.spec.ts`: 4 tests (default mode) + * - `a2.spec.ts`: 2 tests (parallel mode) + * - `a3.spec.ts`: 2 tests (parallel mode) + * - `a4.spec.ts`: 2 tests (default mode) + * + * Test Groups distribution for 2 shards + * ``` + * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] + * Shard 1/2: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^5 + * Shard 2/2: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 ^^^^^^^2 ^^^^^^^2 + * ``` + * + * Test Groups distribution for 3 shards + * ``` + * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] + * Shard 1/3: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 + * Shard 2/3: ^^^^^^^5 ^^^^^^^2 ^^^^^^^2 + * Shard 3/3: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 + * ``` + * + * Test Groups distribution for 4 shards + * ``` + * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] + * Shard 1/4: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 + * Shard 2/4: ^^^^^^^2 ^^^^^^^2 + * Shard 3/4: ^^^^^^^3 ^^^^^^^5 + * Shard 4/4: ^^^^^^^4 ^^^^^^^6 + * ``` + * + * Test Groups distribution for 4 shards with fully-parallel + * ``` + * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] + * Shard 1/4: ^^^^^^^1 ^^^^^^^5 ^^^^^^^9 + * Shard 2/4: ^^^^^^^2 ^^^^^^^6 ^^^^^^10 + * Shard 3/4: ^^^^^^^3 ^^^^^^^7 ^^^^^^11 + * Shard 4/4: ^^^^^^^4 ^^^^^^^8 + * ``` + */ +const tests = { + 'a1.spec.ts': ` + import { test } from '@playwright/test'; + test('test1', async () => { + console.log('\\n%%a1-test1-done'); + }); + test('test2', async () => { + console.log('\\n%%a1-test2-done'); + }); + test('test3', async () => { + console.log('\\n%%a1-test3-done'); + }); + test('test4', async () => { + console.log('\\n%%a1-test4-done'); + }); + `, + 'a2.spec.ts': ` + import { test } from '@playwright/test'; + test.describe.configure({ mode: 'parallel' }); + test('test1', async () => { + console.log('\\n%%a2-test1-done'); + }); + test('test2', async () => { + console.log('\\n%%a2-test2-done'); + }); + `, + 'a3.spec.ts': ` + import { test } from '@playwright/test'; + test.describe.configure({ mode: 'parallel' }); + test('test1', async () => { + console.log('\\n%%a3-test1-done'); + }); + test('test2', async () => { + console.log('\\n%%a3-test2-done'); + }); + `, + 'a4.spec.ts': ` + import { test } from '@playwright/test'; + test('test1', async () => { + console.log('\\n%%a4-test1-done'); + }); + test('test2', async () => { + console.log('\\n%%a4-test2-done'); + }); + `, +}; + +test('should respect shard=1/2', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/2', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(5); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a1-test1-done', + 'a1-test2-done', + 'a1-test3-done', + 'a1-test4-done', + 'a3-test1-done', + ]); +}); + +test('should respect shard=2/2', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(5); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a2-test1-done', + 'a2-test2-done', + 'a3-test2-done', + 'a4-test1-done', + 'a4-test2-done', + ]); +}); + +test('should respect shard=1/3', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/3', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(4); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a1-test1-done', + 'a1-test2-done', + 'a1-test3-done', + 'a1-test4-done', + ]); +}); + +test('should respect shard=2/3', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/3', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a3-test1-done', + 'a4-test1-done', + 'a4-test2-done', + ]); +}); + +test('should respect shard=3/3', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '3/3', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a2-test1-done', + 'a2-test2-done', + 'a3-test2-done', + ]); +}); + +test('should respect shard=1/4', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/4', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(4); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a1-test1-done', + 'a1-test2-done', + 'a1-test3-done', + 'a1-test4-done', + ]); +}); + +test('should respect shard=2/4', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/4', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a4-test1-done', + 'a4-test2-done', + ]); +}); + +test('should respect shard=3/4', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '3/4', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a2-test1-done', + 'a3-test1-done', + ]); +}); + +test('should respect shard=4/4', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '4/4', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a2-test2-done', + 'a3-test2-done', + ]); +}); + +test('should not produce skipped tests for zero-sized shards', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '10/10', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(0); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([]); +}); + +test('should respect shard=1/2 in config', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...tests, + 'playwright.config.js': ` + module.exports = { shardingMode: 'round-robin', shard: { current: 1, total: 2 } }; + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(5); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a1-test1-done', + 'a1-test2-done', + 'a1-test3-done', + 'a1-test4-done', + 'a3-test1-done', + ]); +}); + +test('should work with workers=1 and --fully-parallel', async ({ runInlineTest }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/21226' }); + const tests = { + 'a1.spec.ts': ` + import { test } from '@playwright/test'; + test('should pass', async ({ }) => { + }); + `, + 'a2.spec.ts': ` + import { test } from '@playwright/test'; + test('should pass', async ({ }) => { + }); + test.skip('should skip', async ({ }) => { + }); + `, + }; + + { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/2', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.skipped).toBe(1); + } + { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.skipped).toBe(0); + } +}); + +test('should skip dependency when project is sharded out', async ({ runInlineTest }) => { + const tests = { + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'setup1', testMatch: /setup.ts/ }, + { name: 'tests1', dependencies: ['setup1'] }, + { name: 'setup2', testMatch: /setup.ts/ }, + { name: 'tests2', dependencies: ['setup2'] }, + ], + }; + `, + 'test.spec.ts': ` + import { test } from '@playwright/test'; + test('test', async ({}) => { + console.log('\\n%%test in ' + test.info().project.name); + }); + `, + 'setup.ts': ` + import { test } from '@playwright/test'; + test('setup', async ({}) => { + console.log('\\n%%setup in ' + test.info().project.name); + }); + `, + }; + + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'setup in setup2', + 'test in tests2', + ]); +}); + +test('should not shard mode:default suites', async ({ runInlineTest }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/22891' }); + + const tests = { + 'a1.spec.ts': ` + import { test } from '@playwright/test'; + test('test0', async ({ }) => { + console.log('\\n%%test0'); + }); + test('test1', async ({ }) => { + console.log('\\n%%test1'); + }); + `, + 'a2.spec.ts': ` + import { test } from '@playwright/test'; + test.describe.configure({ mode: 'parallel' }); + + test.describe(() => { + test.describe.configure({ mode: 'default' }); + test.beforeAll(() => { + console.log('\\n%%beforeAll1'); + }); + test('test2', async ({ }) => { + console.log('\\n%%test2'); + }); + test('test3', async ({ }) => { + console.log('\\n%%test3'); + }); + }); + + test.describe(() => { + test.describe.configure({ mode: 'default' }); + test.beforeAll(() => { + console.log('\\n%%beforeAll2'); + }); + test('test4', async ({ }) => { + console.log('\\n%%test4'); + }); + test('test5', async ({ }) => { + console.log('\\n%%test5'); + }); + }); + `, + }; + + { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/3', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.outputLines).toEqual(['beforeAll1', 'test2', 'test3']); + } + { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '3/3', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']); + } +}); + +test('should not shard mode:serial suites when fully-parallel', async ({ runInlineTest }) => { + const tests = { + 'a1.spec.ts': ` + import { test } from '@playwright/test'; + test.describe.configure({ mode: 'serial' }); + test('test1', async ({ }) => { + console.log('\\n%%a1-test1-done'); + }); + test('test2', async ({ }) => { + console.log('\\n%%a1-test2-done'); + }); + test('test3', async ({ }) => { + console.log('\\n%%a1-test3-done'); + }); + `, + 'a2.spec.ts': ` + import { test } from '@playwright/test'; + test.describe(() => { + test('test1', async ({ }) => { + console.log('\\n%%a2-test1-done'); + }); + test('test2', async ({ }) => { + console.log('\\n%%a2-test2-done'); + }); + }); + `, + 'a3.spec.ts': ` + import { test } from '@playwright/test'; + test.describe(() => { + test('test1', async ({ }) => { + console.log('\\n%%a3-test1-done'); + }); + test('test2', async ({ }) => { + console.log('\\n%%a3-test2-done'); + }); + }); + `, + }; + + { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/2', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(4); + expect(result.outputLines).toEqual([ + 'a1-test1-done', + 'a1-test2-done', + 'a1-test3-done', + 'a3-test2-done', + ]); + } + { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.outputLines).toEqual([ + 'a2-test1-done', + 'a2-test2-done', + 'a3-test1-done', + ]); + } + { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/5', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.outputLines).toEqual([ + 'a1-test1-done', + 'a1-test2-done', + 'a1-test3-done', + ]); + } + { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/5', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.outputLines).toEqual([ + 'a2-test1-done', + ]); + } + { + const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '5/5', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.outputLines).toEqual([ + 'a3-test2-done', + ]); + } +}); diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.ts index 9c3839aaa8..e82a434f52 100644 --- a/tests/playwright-test/shard.spec.ts +++ b/tests/playwright-test/shard.spec.ts @@ -14,48 +14,8 @@ * limitations under the License. */ -import { expect, test } from './playwright-test-fixtures'; +import { test, expect } from './playwright-test-fixtures'; -/** - * Test cases: - * - `a1.spec.ts`: 4 tests (default mode) - * - `a2.spec.ts`: 2 tests (parallel mode) - * - `a3.spec.ts`: 2 tests (parallel mode) - * - `a4.spec.ts`: 2 tests (default mode) - * - * Test Groups distribution for 2 shards - * ``` - * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] - * Shard 1/2: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^5 - * Shard 2/2: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 ^^^^^^^2 ^^^^^^^2 - * ``` - * - * Test Groups distribution for 3 shards - * ``` - * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] - * Shard 1/3: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 - * Shard 2/3: ^^^^^^^5 ^^^^^^^2 ^^^^^^^2 - * Shard 3/3: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 - * ``` - * - * Test Groups distribution for 4 shards - * ``` - * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] - * Shard 1/4: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 - * Shard 2/4: ^^^^^^^2 ^^^^^^^2 - * Shard 3/4: ^^^^^^^3 ^^^^^^^5 - * Shard 4/4: ^^^^^^^4 ^^^^^^^6 - * ``` - * - * Test Groups distribution for 4 shards with fully-parallel - * ``` - * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] - * Shard 1/4: ^^^^^^^1 ^^^^^^^5 ^^^^^^^9 - * Shard 2/4: ^^^^^^^2 ^^^^^^^6 ^^^^^^10 - * Shard 3/4: ^^^^^^^3 ^^^^^^^7 ^^^^^^11 - * Shard 4/4: ^^^^^^^4 ^^^^^^^8 - * ``` - */ const tests = { 'a1.spec.ts': ` import { test } from '@playwright/test'; @@ -113,7 +73,7 @@ test('should respect shard=1/2', async ({ runInlineTest }) => { 'a1-test2-done', 'a1-test3-done', 'a1-test4-done', - 'a3-test1-done', + 'a2-test1-done', ]); }); @@ -123,8 +83,8 @@ test('should respect shard=2/2', async ({ runInlineTest }) => { expect(result.passed).toBe(5); expect(result.skipped).toBe(0); expect(result.outputLines).toEqual([ - 'a2-test1-done', 'a2-test2-done', + 'a3-test1-done', 'a3-test2-done', 'a4-test1-done', 'a4-test2-done', @@ -150,9 +110,9 @@ test('should respect shard=2/3', async ({ runInlineTest }) => { expect(result.passed).toBe(3); expect(result.skipped).toBe(0); expect(result.outputLines).toEqual([ + 'a2-test1-done', + 'a2-test2-done', 'a3-test1-done', - 'a4-test1-done', - 'a4-test2-done', ]); }); @@ -162,31 +122,7 @@ test('should respect shard=3/3', async ({ runInlineTest }) => { expect(result.passed).toBe(3); expect(result.skipped).toBe(0); expect(result.outputLines).toEqual([ - 'a2-test1-done', - 'a2-test2-done', 'a3-test2-done', - ]); -}); - -test('should respect shard=1/4', async ({ runInlineTest }) => { - const result = await runInlineTest(tests, { shard: '1/4', workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(4); - expect(result.skipped).toBe(0); - expect(result.outputLines).toEqual([ - 'a1-test1-done', - 'a1-test2-done', - 'a1-test3-done', - 'a1-test4-done', - ]); -}); - -test('should respect shard=2/4', async ({ runInlineTest }) => { - const result = await runInlineTest(tests, { shard: '2/4', workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(2); - expect(result.skipped).toBe(0); - expect(result.outputLines).toEqual([ 'a4-test1-done', 'a4-test2-done', ]); @@ -198,18 +134,7 @@ test('should respect shard=3/4', async ({ runInlineTest }) => { expect(result.passed).toBe(2); expect(result.skipped).toBe(0); expect(result.outputLines).toEqual([ - 'a2-test1-done', 'a3-test1-done', - ]); -}); - -test('should respect shard=4/4', async ({ runInlineTest }) => { - const result = await runInlineTest(tests, { shard: '4/4', workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(2); - expect(result.skipped).toBe(0); - expect(result.outputLines).toEqual([ - 'a2-test2-done', 'a3-test2-done', ]); }); @@ -238,7 +163,7 @@ test('should respect shard=1/2 in config', async ({ runInlineTest }) => { 'a1-test2-done', 'a1-test3-done', 'a1-test4-done', - 'a3-test1-done', + 'a2-test1-done', ]); }); @@ -249,28 +174,20 @@ test('should work with workers=1 and --fully-parallel', async ({ runInlineTest } import { test } from '@playwright/test'; test('should pass', async ({ }) => { }); - `, - 'a2.spec.ts': ` - import { test } from '@playwright/test'; - test('should pass', async ({ }) => { - }); test.skip('should skip', async ({ }) => { }); `, + 'a2.spec.ts': ` + import { test } from '@playwright/test'; + test('should pass', async ({ }) => { + }); + `, }; - { - const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.skipped).toBe(1); - } - { - const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.skipped).toBe(0); - } + const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.skipped).toBe(1); }); test('should skip dependency when project is sharded out', async ({ runInlineTest }) => { @@ -367,91 +284,3 @@ test('should not shard mode:default suites', async ({ runInlineTest }) => { expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']); } }); - -test('should not shard mode:serial suites when fully-parallel', async ({ runInlineTest }) => { - const tests = { - 'a1.spec.ts': ` - import { test } from '@playwright/test'; - test.describe.configure({ mode: 'serial' }); - test('test1', async ({ }) => { - console.log('\\n%%a1-test1-done'); - }); - test('test2', async ({ }) => { - console.log('\\n%%a1-test2-done'); - }); - test('test3', async ({ }) => { - console.log('\\n%%a1-test3-done'); - }); - `, - 'a2.spec.ts': ` - import { test } from '@playwright/test'; - test.describe(() => { - test('test1', async ({ }) => { - console.log('\\n%%a2-test1-done'); - }); - test('test2', async ({ }) => { - console.log('\\n%%a2-test2-done'); - }); - }); - `, - 'a3.spec.ts': ` - import { test } from '@playwright/test'; - test.describe(() => { - test('test1', async ({ }) => { - console.log('\\n%%a3-test1-done'); - }); - test('test2', async ({ }) => { - console.log('\\n%%a3-test2-done'); - }); - }); - `, - }; - - { - const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(4); - expect(result.outputLines).toEqual([ - 'a1-test1-done', - 'a1-test2-done', - 'a1-test3-done', - 'a3-test2-done', - ]); - } - { - const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(3); - expect(result.outputLines).toEqual([ - 'a2-test1-done', - 'a2-test2-done', - 'a3-test1-done', - ]); - } - { - const result = await runInlineTest(tests, { shard: '1/5', ['fully-parallel']: true, workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(3); - expect(result.outputLines).toEqual([ - 'a1-test1-done', - 'a1-test2-done', - 'a1-test3-done', - ]); - } - { - const result = await runInlineTest(tests, { shard: '2/5', ['fully-parallel']: true, workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.outputLines).toEqual([ - 'a2-test1-done', - ]); - } - { - const result = await runInlineTest(tests, { shard: '5/5', ['fully-parallel']: true, workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.outputLines).toEqual([ - 'a3-test2-done', - ]); - } -});