Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add shardingMode configuration

Browse source
This commit is contained in:
Mathias Leppich 2024-05-27 15:47:04 +02:00
parent 14ca30ae12
commit ce09f88d76
9 changed files with 601 additions and 196 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
docs/src/test-api/class-testconfig.md
View file

@ -482,6 +482,25 @@ export default defineConfig({
``` ```
## property: TestConfig.shardingMode
* since: v1.45
- type: ?<[ShardingMode]<"partition"|"round-robin"|"duration-round-robin">>
Defines the algorithm to be used for sharding. Defaults to `'partition'`.
* `'partition'` - divide the set of test groups by number of shards. e.g. first
half goes to shard 1/2 and seconds half to shard 2/2.
* `'round-robin'` - spread test groups to shards in a round-robin way. e.g. loop
over test groups and always assign to the shard that has the lowest number of
tests.
* `'duration-round-robin'` - use duration info from `.last-run.json` to spread
test groups to shards in a round-robin way. e.g. loop over test groups and
always assign to the shard that has the lowest duration of tests. new tests
which were not present in the last run will use an average duration time. When
no `.last-run.json` could be found the behavior is identical to
`'round-robin'`.
## property: TestConfig.shardingSeed ## property: TestConfig.shardingSeed
* since: v1.45 * since: v1.45

6
packages/playwright/src/common/config.ts
View file

@ -17,7 +17,7 @@
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import os from 'os'; import os from 'os';
import type { Config, Fixtures, Project, ReporterDescription } from '../../types/test'; import type { Config, Fixtures, PlaywrightTestConfig, Project, ReporterDescription } from '../../types/test';
import type { Location } from '../../types/testReporter'; import type { Location } from '../../types/testReporter';
import type { TestRunnerPluginRegistration } from '../plugins'; import type { TestRunnerPluginRegistration } from '../plugins';
import { getPackageJsonPath, mergeObjects } from '../util'; import { getPackageJsonPath, mergeObjects } from '../util';
@ -25,6 +25,7 @@ import type { Matcher } from '../util';
import type { ConfigCLIOverrides } from './ipc'; import type { ConfigCLIOverrides } from './ipc';
import type { FullConfig, FullProject } from '../../types/testReporter'; import type { FullConfig, FullProject } from '../../types/testReporter';
import { setTransformConfig } from '../transform/transform'; import { setTransformConfig } from '../transform/transform';
import type { LastRunInfo } from '../runner/runner';
export type ConfigLocation = { export type ConfigLocation = {
resolvedConfigFile?: string; resolvedConfigFile?: string;
@ -55,7 +56,9 @@ export class FullConfigInternal {
cliFailOnFlakyTests?: boolean; cliFailOnFlakyTests?: boolean;
testIdMatcher?: Matcher; testIdMatcher?: Matcher;
defineConfigWasUsed = false; defineConfigWasUsed = false;
shardingMode: Exclude<PlaywrightTestConfig['shardingMode'], undefined>;
shardingSeed: string | null; shardingSeed: string | null;
lastRunInfo?: LastRunInfo;
constructor(location: ConfigLocation, userConfig: Config, configCLIOverrides: ConfigCLIOverrides) { constructor(location: ConfigLocation, userConfig: Config, configCLIOverrides: ConfigCLIOverrides) {
if (configCLIOverrides.projects && userConfig.projects) if (configCLIOverrides.projects && userConfig.projects)
@ -93,6 +96,7 @@ export class FullConfigInternal {
workers: 0, workers: 0,
webServer: null, webServer: null,
}; };
this.shardingMode = takeFirst(configCLIOverrides.shardingMode, userConfig.shardingMode, 'partition');
this.shardingSeed = takeFirst(configCLIOverrides.shardingSeed, userConfig.shardingSeed, null); this.shardingSeed = takeFirst(configCLIOverrides.shardingSeed, userConfig.shardingSeed, null);
for (const key in userConfig) { for (const key in userConfig) {
if (key.startsWith('@')) if (key.startsWith('@'))

3
packages/playwright/src/common/ipc.ts
View file

@ -17,7 +17,7 @@
import util from 'util'; import util from 'util';
import { type SerializedCompilationCache, serializeCompilationCache } from '../transform/compilationCache'; import { type SerializedCompilationCache, serializeCompilationCache } from '../transform/compilationCache';
import type { ConfigLocation, FullConfigInternal } from './config'; import type { ConfigLocation, FullConfigInternal } from './config';
import type { ReporterDescription, TestInfoError, TestStatus } from '../../types/test'; import type { PlaywrightTestConfig, ReporterDescription, TestInfoError, TestStatus } from '../../types/test';
export type ConfigCLIOverrides = { export type ConfigCLIOverrides = {
forbidOnly?: boolean; forbidOnly?: boolean;
@ -32,6 +32,7 @@ export type ConfigCLIOverrides = {
reporter?: ReporterDescription[]; reporter?: ReporterDescription[];
additionalReporters?: ReporterDescription[]; additionalReporters?: ReporterDescription[];
shard?: { current: number, total: number }; shard?: { current: number, total: number };
shardingMode?: PlaywrightTestConfig['shardingMode'];
shardingSeed?: string; shardingSeed?: string;
timeout?: number; timeout?: number;
ignoreSnapshots?: boolean; ignoreSnapshots?: boolean;

8
packages/playwright/src/program.ts
View file

@ -184,9 +184,13 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
if (!config) if (!config)
return; return;
if (opts.lastFailed) { if (opts.lastFailed || config.shardingMode === 'duration-round-robin') {
const lastRunInfo = await readLastRunInfo(config); const lastRunInfo = await readLastRunInfo(config);
if (opts.lastFailed)
config.testIdMatcher = id => lastRunInfo.failedTests.includes(id); config.testIdMatcher = id => lastRunInfo.failedTests.includes(id);
if (config.shardingMode === 'duration-round-robin')
config.lastRunInfo = lastRunInfo;
} }
config.cliArgs = args; config.cliArgs = args;
@ -281,6 +285,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
retries: options.retries ? parseInt(options.retries, 10) : undefined, retries: options.retries ? parseInt(options.retries, 10) : undefined,
reporter: resolveReporterOption(options.reporter), reporter: resolveReporterOption(options.reporter),
shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined, shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined,
shardingMode: options.shardingMode ? options.shardingMode : undefined,
shardingSeed: options.shardingSeed ? options.shardingSeed : undefined, shardingSeed: options.shardingSeed ? options.shardingSeed : undefined,
timeout: options.timeout ? parseInt(options.timeout, 10) : undefined, timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined, ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
@ -359,6 +364,7 @@ const testOptions: [string, string][] = [
['--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`], ['--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`],
['--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`], ['--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`],
['--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`], ['--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`],
['--sharding-mode <mode>', `Sharding algorithm to use; "partition", "round-robin" or "duration-round-robin". Defaults to "partition".`],
['--sharding-seed <seed>', `Seed string for randomizing the test order before sharding. Defaults to not randomizing the order.`], ['--sharding-seed <seed>', `Seed string for randomizing the test order before sharding. Defaults to not randomizing the order.`],
['--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`], ['--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`],
['--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`], ['--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`],

2
packages/playwright/src/runner/loadUtils.ts
View file

@ -184,7 +184,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
shuffleWithSeed(testGroups, config.shardingSeed); shuffleWithSeed(testGroups, config.shardingSeed);
// Shard test groups. // Shard test groups.
const testGroupsInThisShard = filterForShard(config.config.shard, testGroups); const testGroupsInThisShard = filterForShard(config.shardingMode, config.config.shard, testGroups, config.lastRunInfo);
const testsInThisShard = new Set<TestCase>(); const testsInThisShard = new Set<TestCase>();
for (const group of testGroupsInThisShard) { for (const group of testGroupsInThisShard) {
for (const test of group.tests) for (const test of group.tests)

86
packages/playwright/src/runner/testGroups.ts
View file

@ -14,7 +14,9 @@
* limitations under the License. * limitations under the License.
*/ */
import type { PlaywrightTestConfig } from '../../types/test';
import type { Suite, TestCase } from '../common/test'; import type { Suite, TestCase } from '../common/test';
import type { LastRunInfo } from './runner';
export type TestGroup = { export type TestGroup = {
workerHash: string; workerHash: string;
@ -130,7 +132,12 @@ export function createTestGroups(projectSuite: Suite, workers: number): TestGrou
return result; return result;
} }
export function filterForShard(shard: { total: number, current: number }, testGroups: TestGroup[]): Set<TestGroup> { export function filterForShard(
mode: PlaywrightTestConfig['shardingMode'],
shard: { total: number, current: number },
testGroups: TestGroup[],
lastRunInfo?: LastRunInfo,
): Set<TestGroup> {
// Note that sharding works based on test groups. // Note that sharding works based on test groups.
// This means parallel files will be sharded by single tests, // This means parallel files will be sharded by single tests,
// while non-parallel files will be sharded by the whole file. // while non-parallel files will be sharded by the whole file.
@ -138,16 +145,85 @@ export function filterForShard(shard: { total: number, current: number }, testGr
// Shards are still balanced by the number of tests, not files, // Shards are still balanced by the number of tests, not files,
// even in the case of non-paralleled files. // even in the case of non-paralleled files.
const lengths = new Array(shard.total).fill(0); if (mode === 'round-robin')
return filterForShardRoundRobin(shard, testGroups);
if (mode === 'duration-round-robin')
return filterForShardRoundRobin(shard, testGroups, lastRunInfo);
return filterForShardPartition(shard, testGroups);
}
/**
* Shards tests by partitioning them into equal parts.
*
* ```
* [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
* Shard 1: ^---------^ : [ 1, 2, 3 ]
* Shard 2: ^---------^ : [ 4, 5, 6 ]
* Shard 3: ^---------^ : [ 7, 8, 9 ]
* Shard 4: ^---------^ : [ 10,11,12 ]
* ```
*/
function filterForShardPartition(shard: { total: number, current: number }, testGroups: TestGroup[]): Set<TestGroup> {
let shardableTotal = 0;
for (const group of testGroups)
shardableTotal += group.tests.length;
// Each shard gets some tests.
const shardSize = Math.floor(shardableTotal / shard.total);
// First few shards get one more test each.
const extraOne = shardableTotal - shardSize * shard.total;
const currentShard = shard.current - 1; // Make it zero-based for calculations.
const from = shardSize * currentShard + Math.min(extraOne, currentShard);
const to = from + shardSize + (currentShard < extraOne ? 1 : 0);
let current = 0;
const result = new Set<TestGroup>();
for (const group of testGroups) {
// Any test group goes to the shard that contains the first test of this group.
// So, this shard gets any group that starts at [from; to)
if (current >= from && current < to)
result.add(group);
current += group.tests.length;
}
return result;
}
/**
* Shards tests by round-robin.
*
* ```
* [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
* Shard 1: ^ ^ ^ : [ 1, 5, 9 ]
* Shard 2: ^ ^ ^ : [ 2, 6,10 ]
* Shard 3: ^ ^ ^ : [ 3, 7,11 ]
* Shard 4: ^ ^ ^ : [ 4, 8,12 ]
* ```
*/
function filterForShardRoundRobin(
shard: { total: number, current: number },
testGroups: TestGroup[],
lastRunInfo?: LastRunInfo,
): Set<TestGroup> {
const weights = new Array(shard.total).fill(0);
const shardSet = new Array(shard.total).fill(0).map(() => new Set<TestGroup>()); const shardSet = new Array(shard.total).fill(0).map(() => new Set<TestGroup>());
const averageDuration = lastRunInfo ? Object.values(lastRunInfo?.testDurations || {}).reduce((a, b) => a + b, 1) / Math.max(1, Object.values(lastRunInfo?.testDurations || {}).length) : 0;
const weight = (group: TestGroup) => {
if (!lastRunInfo)
// If we don't have last run info, we just count the number of tests.
return group.tests.length;
// If we have last run info, we use the duration of the tests.
return group.tests.reduce((sum, test) => sum + Math.max(1, lastRunInfo.testDurations?.[test.id] || averageDuration), 0);
};
// We sort the test groups by the number of tests in descending order. // We sort the test groups by the number of tests in descending order.
const sortedTestGroups = testGroups.slice().sort((a, b) => b.tests.length - a.tests.length); const sortedTestGroups = testGroups.slice().sort((a, b) => weight(b) - weight(a));
// Then we add each group to the shard with the smallest number of tests. // Then we add each group to the shard with the smallest number of tests.
for (const group of sortedTestGroups) { for (const group of sortedTestGroups) {
const index = lengths.reduce((minIndex, currentLength, currentIndex) => currentLength < lengths[minIndex] ? currentIndex : minIndex, 0); const index = weights.reduce((minIndex, currentLength, currentIndex) => currentLength < weights[minIndex] ? currentIndex : minIndex, 0);
lengths[index] += group.tests.length; weights[index] += weight(group);
shardSet[index].add(group); shardSet[index].add(group);
} }

13
packages/playwright/types/test.d.ts vendored
View file

@ -1425,6 +1425,19 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
total: number; total: number;
}; };
/**
* Defines the algorithm to be used for sharding. Defaults to `'partition'`.
* - `'partition'` - divide the set of test groups by number of shards. e.g. first half goes to shard 1/2 and
* seconds half to shard 2/2.
* - `'round-robin'` - spread test groups to shards in a round-robin way. e.g. loop over test groups and always
* assign to the shard that has the lowest number of tests.
* - `'duration-round-robin'` - use duration info from `.last-run.json` to spread test groups to shards in a
* round-robin way. e.g. loop over test groups and always assign to the shard that has the lowest duration of
* tests. new tests which were not present in the last run will use an average duration time. When no
* `.last-run.json` could be found the behavior is identical to `'round-robin'`.
*/
shardingMode?: "partition"|"round-robin"|"duration-round-robin";
/** /**
* Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is * Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is
* mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests * mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests

457
tests/playwright-test/shard-roundrobin.spec.ts Normal file
View file

@ -0,0 +1,457 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { expect, test } from './playwright-test-fixtures';
/**
* Test cases:
* - `a1.spec.ts`: 4 tests (default mode)
* - `a2.spec.ts`: 2 tests (parallel mode)
* - `a3.spec.ts`: 2 tests (parallel mode)
* - `a4.spec.ts`: 2 tests (default mode)
*
* Test Groups distribution for 2 shards
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/2: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^5
* Shard 2/2: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 ^^^^^^^2 ^^^^^^^2
* ```
*
* Test Groups distribution for 3 shards
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/3: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
* Shard 2/3: ^^^^^^^5 ^^^^^^^2 ^^^^^^^2
* Shard 3/3: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6
* ```
*
* Test Groups distribution for 4 shards
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/4: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
* Shard 2/4: ^^^^^^^2 ^^^^^^^2
* Shard 3/4: ^^^^^^^3 ^^^^^^^5
* Shard 4/4: ^^^^^^^4 ^^^^^^^6
* ```
*
* Test Groups distribution for 4 shards with fully-parallel
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/4: ^^^^^^^1 ^^^^^^^5 ^^^^^^^9
* Shard 2/4: ^^^^^^^2 ^^^^^^^6 ^^^^^^10
* Shard 3/4: ^^^^^^^3 ^^^^^^^7 ^^^^^^11
* Shard 4/4: ^^^^^^^4 ^^^^^^^8
* ```
*/
const tests = {
'a1.spec.ts': `
import { test } from '@playwright/test';
test('test1', async () => {
console.log('\\n%%a1-test1-done');
});
test('test2', async () => {
console.log('\\n%%a1-test2-done');
});
test('test3', async () => {
console.log('\\n%%a1-test3-done');
});
test('test4', async () => {
console.log('\\n%%a1-test4-done');
});
`,
'a2.spec.ts': `
import { test } from '@playwright/test';
test.describe.configure({ mode: 'parallel' });
test('test1', async () => {
console.log('\\n%%a2-test1-done');
});
test('test2', async () => {
console.log('\\n%%a2-test2-done');
});
`,
'a3.spec.ts': `
import { test } from '@playwright/test';
test.describe.configure({ mode: 'parallel' });
test('test1', async () => {
console.log('\\n%%a3-test1-done');
});
test('test2', async () => {
console.log('\\n%%a3-test2-done');
});
`,
'a4.spec.ts': `
import { test } from '@playwright/test';
test('test1', async () => {
console.log('\\n%%a4-test1-done');
});
test('test2', async () => {
console.log('\\n%%a4-test2-done');
});
`,
};
test('should respect shard=1/2', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/2', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(5);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
'a1-test4-done',
'a3-test1-done',
]);
});
test('should respect shard=2/2', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(5);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test2-done',
'a4-test1-done',
'a4-test2-done',
]);
});
test('should respect shard=1/3', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/3', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
'a1-test4-done',
]);
});
test('should respect shard=2/3', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/3', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a3-test1-done',
'a4-test1-done',
'a4-test2-done',
]);
});
test('should respect shard=3/3', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '3/3', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test2-done',
]);
});
test('should respect shard=1/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
'a1-test4-done',
]);
});
test('should respect shard=2/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a4-test1-done',
'a4-test2-done',
]);
});
test('should respect shard=3/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '3/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a3-test1-done',
]);
});
test('should respect shard=4/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '4/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test2-done',
'a3-test2-done',
]);
});
test('should not produce skipped tests for zero-sized shards', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '10/10', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(0);
expect(result.failed).toBe(0);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([]);
});
test('should respect shard=1/2 in config', async ({ runInlineTest }) => {
const result = await runInlineTest({
...tests,
'playwright.config.js': `
module.exports = { shardingMode: 'round-robin', shard: { current: 1, total: 2 } };
`,
}, { workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(5);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
'a1-test4-done',
'a3-test1-done',
]);
});
test('should work with workers=1 and --fully-parallel', async ({ runInlineTest }) => {
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/21226' });
const tests = {
'a1.spec.ts': `
import { test } from '@playwright/test';
test('should pass', async ({ }) => {
});
`,
'a2.spec.ts': `
import { test } from '@playwright/test';
test('should pass', async ({ }) => {
});
test.skip('should skip', async ({ }) => {
});
`,
};
{
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.skipped).toBe(1);
}
{
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.skipped).toBe(0);
}
});
test('should skip dependency when project is sharded out', async ({ runInlineTest }) => {
const tests = {
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'setup1', testMatch: /setup.ts/ },
{ name: 'tests1', dependencies: ['setup1'] },
{ name: 'setup2', testMatch: /setup.ts/ },
{ name: 'tests2', dependencies: ['setup2'] },
],
};
`,
'test.spec.ts': `
import { test } from '@playwright/test';
test('test', async ({}) => {
console.log('\\n%%test in ' + test.info().project.name);
});
`,
'setup.ts': `
import { test } from '@playwright/test';
test('setup', async ({}) => {
console.log('\\n%%setup in ' + test.info().project.name);
});
`,
};
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'setup in setup2',
'test in tests2',
]);
});
test('should not shard mode:default suites', async ({ runInlineTest }) => {
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/22891' });
const tests = {
'a1.spec.ts': `
import { test } from '@playwright/test';
test('test0', async ({ }) => {
console.log('\\n%%test0');
});
test('test1', async ({ }) => {
console.log('\\n%%test1');
});
`,
'a2.spec.ts': `
import { test } from '@playwright/test';
test.describe.configure({ mode: 'parallel' });
test.describe(() => {
test.describe.configure({ mode: 'default' });
test.beforeAll(() => {
console.log('\\n%%beforeAll1');
});
test('test2', async ({ }) => {
console.log('\\n%%test2');
});
test('test3', async ({ }) => {
console.log('\\n%%test3');
});
});
test.describe(() => {
test.describe.configure({ mode: 'default' });
test.beforeAll(() => {
console.log('\\n%%beforeAll2');
});
test('test4', async ({ }) => {
console.log('\\n%%test4');
});
test('test5', async ({ }) => {
console.log('\\n%%test5');
});
});
`,
};
{
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/3', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.outputLines).toEqual(['beforeAll1', 'test2', 'test3']);
}
{
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '3/3', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']);
}
});
test('should not shard mode:serial suites when fully-parallel', async ({ runInlineTest }) => {
const tests = {
'a1.spec.ts': `
import { test } from '@playwright/test';
test.describe.configure({ mode: 'serial' });
test('test1', async ({ }) => {
console.log('\\n%%a1-test1-done');
});
test('test2', async ({ }) => {
console.log('\\n%%a1-test2-done');
});
test('test3', async ({ }) => {
console.log('\\n%%a1-test3-done');
});
`,
'a2.spec.ts': `
import { test } from '@playwright/test';
test.describe(() => {
test('test1', async ({ }) => {
console.log('\\n%%a2-test1-done');
});
test('test2', async ({ }) => {
console.log('\\n%%a2-test2-done');
});
});
`,
'a3.spec.ts': `
import { test } from '@playwright/test';
test.describe(() => {
test('test1', async ({ }) => {
console.log('\\n%%a3-test1-done');
});
test('test2', async ({ }) => {
console.log('\\n%%a3-test2-done');
});
});
`,
};
{
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
'a3-test2-done',
]);
}
{
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test1-done',
]);
}
{
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/5', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
]);
}
{
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/5', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.outputLines).toEqual([
'a2-test1-done',
]);
}
{
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '5/5', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.outputLines).toEqual([
'a3-test2-done',
]);
}
});

187
tests/playwright-test/shard.spec.ts
View file

@ -14,48 +14,8 @@
* limitations under the License. * limitations under the License.
*/ */
import { expect, test } from './playwright-test-fixtures'; import { test, expect } from './playwright-test-fixtures';
/**
* Test cases:
* - `a1.spec.ts`: 4 tests (default mode)
* - `a2.spec.ts`: 2 tests (parallel mode)
* - `a3.spec.ts`: 2 tests (parallel mode)
* - `a4.spec.ts`: 2 tests (default mode)
*
* Test Groups distribution for 2 shards
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/2: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^5
* Shard 2/2: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 ^^^^^^^2 ^^^^^^^2
* ```
*
* Test Groups distribution for 3 shards
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/3: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
* Shard 2/3: ^^^^^^^5 ^^^^^^^2 ^^^^^^^2
* Shard 3/3: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6
* ```
*
* Test Groups distribution for 4 shards
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/4: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
* Shard 2/4: ^^^^^^^2 ^^^^^^^2
* Shard 3/4: ^^^^^^^3 ^^^^^^^5
* Shard 4/4: ^^^^^^^4 ^^^^^^^6
* ```
*
* Test Groups distribution for 4 shards with fully-parallel
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/4: ^^^^^^^1 ^^^^^^^5 ^^^^^^^9
* Shard 2/4: ^^^^^^^2 ^^^^^^^6 ^^^^^^10
* Shard 3/4: ^^^^^^^3 ^^^^^^^7 ^^^^^^11
* Shard 4/4: ^^^^^^^4 ^^^^^^^8
* ```
*/
const tests = { const tests = {
'a1.spec.ts': ` 'a1.spec.ts': `
import { test } from '@playwright/test'; import { test } from '@playwright/test';
@ -113,7 +73,7 @@ test('should respect shard=1/2', async ({ runInlineTest }) => {
'a1-test2-done', 'a1-test2-done',
'a1-test3-done', 'a1-test3-done',
'a1-test4-done', 'a1-test4-done',
'a3-test1-done', 'a2-test1-done',
]); ]);
}); });
@ -123,8 +83,8 @@ test('should respect shard=2/2', async ({ runInlineTest }) => {
expect(result.passed).toBe(5); expect(result.passed).toBe(5);
expect(result.skipped).toBe(0); expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([ expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done', 'a2-test2-done',
'a3-test1-done',
'a3-test2-done', 'a3-test2-done',
'a4-test1-done', 'a4-test1-done',
'a4-test2-done', 'a4-test2-done',
@ -150,9 +110,9 @@ test('should respect shard=2/3', async ({ runInlineTest }) => {
expect(result.passed).toBe(3); expect(result.passed).toBe(3);
expect(result.skipped).toBe(0); expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([ expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test1-done', 'a3-test1-done',
'a4-test1-done',
'a4-test2-done',
]); ]);
}); });
@ -162,31 +122,7 @@ test('should respect shard=3/3', async ({ runInlineTest }) => {
expect(result.passed).toBe(3); expect(result.passed).toBe(3);
expect(result.skipped).toBe(0); expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([ expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test2-done', 'a3-test2-done',
]);
});
test('should respect shard=1/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { shard: '1/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
'a1-test4-done',
]);
});
test('should respect shard=2/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { shard: '2/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a4-test1-done', 'a4-test1-done',
'a4-test2-done', 'a4-test2-done',
]); ]);
@ -198,18 +134,7 @@ test('should respect shard=3/4', async ({ runInlineTest }) => {
expect(result.passed).toBe(2); expect(result.passed).toBe(2);
expect(result.skipped).toBe(0); expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([ expect(result.outputLines).toEqual([
'a2-test1-done',
'a3-test1-done', 'a3-test1-done',
]);
});
test('should respect shard=4/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { shard: '4/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test2-done',
'a3-test2-done', 'a3-test2-done',
]); ]);
}); });
@ -238,7 +163,7 @@ test('should respect shard=1/2 in config', async ({ runInlineTest }) => {
'a1-test2-done', 'a1-test2-done',
'a1-test3-done', 'a1-test3-done',
'a1-test4-done', 'a1-test4-done',
'a3-test1-done', 'a2-test1-done',
]); ]);
}); });
@ -249,28 +174,20 @@ test('should work with workers=1 and --fully-parallel', async ({ runInlineTest }
import { test } from '@playwright/test'; import { test } from '@playwright/test';
test('should pass', async ({ }) => { test('should pass', async ({ }) => {
}); });
test.skip('should skip', async ({ }) => {
});
`, `,
'a2.spec.ts': ` 'a2.spec.ts': `
import { test } from '@playwright/test'; import { test } from '@playwright/test';
test('should pass', async ({ }) => { test('should pass', async ({ }) => {
}); });
test.skip('should skip', async ({ }) => {
});
`, `,
}; };
{
const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 }); const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1); expect(result.passed).toBe(1);
expect(result.skipped).toBe(1); expect(result.skipped).toBe(1);
}
{
const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.skipped).toBe(0);
}
}); });
test('should skip dependency when project is sharded out', async ({ runInlineTest }) => { test('should skip dependency when project is sharded out', async ({ runInlineTest }) => {
@ -367,91 +284,3 @@ test('should not shard mode:default suites', async ({ runInlineTest }) => {
expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']); expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']);
} }
}); });
test('should not shard mode:serial suites when fully-parallel', async ({ runInlineTest }) => {
const tests = {
'a1.spec.ts': `
import { test } from '@playwright/test';
test.describe.configure({ mode: 'serial' });
test('test1', async ({ }) => {
console.log('\\n%%a1-test1-done');
});
test('test2', async ({ }) => {
console.log('\\n%%a1-test2-done');
});
test('test3', async ({ }) => {
console.log('\\n%%a1-test3-done');
});
`,
'a2.spec.ts': `
import { test } from '@playwright/test';
test.describe(() => {
test('test1', async ({ }) => {
console.log('\\n%%a2-test1-done');
});
test('test2', async ({ }) => {
console.log('\\n%%a2-test2-done');
});
});
`,
'a3.spec.ts': `
import { test } from '@playwright/test';
test.describe(() => {
test('test1', async ({ }) => {
console.log('\\n%%a3-test1-done');
});
test('test2', async ({ }) => {
console.log('\\n%%a3-test2-done');
});
});
`,
};
{
const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
'a3-test2-done',
]);
}
{
const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test1-done',
]);
}
{
const result = await runInlineTest(tests, { shard: '1/5', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
]);
}
{
const result = await runInlineTest(tests, { shard: '2/5', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.outputLines).toEqual([
'a2-test1-done',
]);
}
{
const result = await runInlineTest(tests, { shard: '5/5', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.outputLines).toEqual([
'a3-test2-done',
]);
}
});