Add shardingMode configuration
This commit is contained in:
Mathias Leppich
parent
14ca30ae12
commit
ce09f88d76
|
|
@ -482,6 +482,25 @@ export default defineConfig({
|
|||
```
|
||||
|
||||
|
||||
## property: TestConfig.shardingMode
|
||||
|
||||
* since: v1.45
|
||||
- type: ?<[ShardingMode]<"partition"|"round-robin"|"duration-round-robin">>
|
||||
|
||||
Defines the algorithm to be used for sharding. Defaults to `'partition'`.
|
||||
* `'partition'` - divide the set of test groups by number of shards. e.g. first
|
||||
half goes to shard 1/2 and seconds half to shard 2/2.
|
||||
* `'round-robin'` - spread test groups to shards in a round-robin way. e.g. loop
|
||||
over test groups and always assign to the shard that has the lowest number of
|
||||
tests.
|
||||
* `'duration-round-robin'` - use duration info from `.last-run.json` to spread
|
||||
test groups to shards in a round-robin way. e.g. loop over test groups and
|
||||
always assign to the shard that has the lowest duration of tests. new tests
|
||||
which were not present in the last run will use an average duration time. When
|
||||
no `.last-run.json` could be found the behavior is identical to
|
||||
`'round-robin'`.
|
||||
|
||||
|
||||
## property: TestConfig.shardingSeed
|
||||
|
||||
* since: v1.45
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
import type { Config, Fixtures, Project, ReporterDescription } from '../../types/test';
|
||||
import type { Config, Fixtures, PlaywrightTestConfig, Project, ReporterDescription } from '../../types/test';
|
||||
import type { Location } from '../../types/testReporter';
|
||||
import type { TestRunnerPluginRegistration } from '../plugins';
|
||||
import { getPackageJsonPath, mergeObjects } from '../util';
|
||||
|
|
@ -25,6 +25,7 @@ import type { Matcher } from '../util';
|
|||
import type { ConfigCLIOverrides } from './ipc';
|
||||
import type { FullConfig, FullProject } from '../../types/testReporter';
|
||||
import { setTransformConfig } from '../transform/transform';
|
||||
import type { LastRunInfo } from '../runner/runner';
|
||||
|
||||
export type ConfigLocation = {
|
||||
resolvedConfigFile?: string;
|
||||
|
|
@ -55,7 +56,9 @@ export class FullConfigInternal {
|
|||
cliFailOnFlakyTests?: boolean;
|
||||
testIdMatcher?: Matcher;
|
||||
defineConfigWasUsed = false;
|
||||
shardingMode: Exclude<PlaywrightTestConfig['shardingMode'], undefined>;
|
||||
shardingSeed: string | null;
|
||||
lastRunInfo?: LastRunInfo;
|
||||
|
||||
constructor(location: ConfigLocation, userConfig: Config, configCLIOverrides: ConfigCLIOverrides) {
|
||||
if (configCLIOverrides.projects && userConfig.projects)
|
||||
|
|
@ -93,6 +96,7 @@ export class FullConfigInternal {
|
|||
workers: 0,
|
||||
webServer: null,
|
||||
};
|
||||
this.shardingMode = takeFirst(configCLIOverrides.shardingMode, userConfig.shardingMode, 'partition');
|
||||
this.shardingSeed = takeFirst(configCLIOverrides.shardingSeed, userConfig.shardingSeed, null);
|
||||
for (const key in userConfig) {
|
||||
if (key.startsWith('@'))
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
import util from 'util';
|
||||
import { type SerializedCompilationCache, serializeCompilationCache } from '../transform/compilationCache';
|
||||
import type { ConfigLocation, FullConfigInternal } from './config';
|
||||
import type { ReporterDescription, TestInfoError, TestStatus } from '../../types/test';
|
||||
import type { PlaywrightTestConfig, ReporterDescription, TestInfoError, TestStatus } from '../../types/test';
|
||||
|
||||
export type ConfigCLIOverrides = {
|
||||
forbidOnly?: boolean;
|
||||
|
|
@ -32,6 +32,7 @@ export type ConfigCLIOverrides = {
|
|||
reporter?: ReporterDescription[];
|
||||
additionalReporters?: ReporterDescription[];
|
||||
shard?: { current: number, total: number };
|
||||
shardingMode?: PlaywrightTestConfig['shardingMode'];
|
||||
shardingSeed?: string;
|
||||
timeout?: number;
|
||||
ignoreSnapshots?: boolean;
|
||||
|
|
|
|||
|
|
@ -184,9 +184,13 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
|
|||
if (!config)
|
||||
return;
|
||||
|
||||
if (opts.lastFailed) {
|
||||
if (opts.lastFailed || config.shardingMode === 'duration-round-robin') {
|
||||
const lastRunInfo = await readLastRunInfo(config);
|
||||
if (opts.lastFailed)
|
||||
config.testIdMatcher = id => lastRunInfo.failedTests.includes(id);
|
||||
|
||||
if (config.shardingMode === 'duration-round-robin')
|
||||
config.lastRunInfo = lastRunInfo;
|
||||
}
|
||||
|
||||
config.cliArgs = args;
|
||||
|
|
@ -281,6 +285,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
|
|||
retries: options.retries ? parseInt(options.retries, 10) : undefined,
|
||||
reporter: resolveReporterOption(options.reporter),
|
||||
shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined,
|
||||
shardingMode: options.shardingMode ? options.shardingMode : undefined,
|
||||
shardingSeed: options.shardingSeed ? options.shardingSeed : undefined,
|
||||
timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
|
||||
ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
|
||||
|
|
@ -359,6 +364,7 @@ const testOptions: [string, string][] = [
|
|||
['--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`],
|
||||
['--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`],
|
||||
['--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`],
|
||||
['--sharding-mode <mode>', `Sharding algorithm to use; "partition", "round-robin" or "duration-round-robin". Defaults to "partition".`],
|
||||
['--sharding-seed <seed>', `Seed string for randomizing the test order before sharding. Defaults to not randomizing the order.`],
|
||||
['--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`],
|
||||
['--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`],
|
||||
|
|
|
|||
|
|
@ -184,7 +184,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
|
|||
shuffleWithSeed(testGroups, config.shardingSeed);
|
||||
|
||||
// Shard test groups.
|
||||
const testGroupsInThisShard = filterForShard(config.config.shard, testGroups);
|
||||
const testGroupsInThisShard = filterForShard(config.shardingMode, config.config.shard, testGroups, config.lastRunInfo);
|
||||
const testsInThisShard = new Set<TestCase>();
|
||||
for (const group of testGroupsInThisShard) {
|
||||
for (const test of group.tests)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,9 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import type { PlaywrightTestConfig } from '../../types/test';
|
||||
import type { Suite, TestCase } from '../common/test';
|
||||
import type { LastRunInfo } from './runner';
|
||||
|
||||
export type TestGroup = {
|
||||
workerHash: string;
|
||||
|
|
@ -130,7 +132,12 @@ export function createTestGroups(projectSuite: Suite, workers: number): TestGrou
|
|||
return result;
|
||||
}
|
||||
|
||||
export function filterForShard(shard: { total: number, current: number }, testGroups: TestGroup[]): Set<TestGroup> {
|
||||
export function filterForShard(
|
||||
mode: PlaywrightTestConfig['shardingMode'],
|
||||
shard: { total: number, current: number },
|
||||
testGroups: TestGroup[],
|
||||
lastRunInfo?: LastRunInfo,
|
||||
): Set<TestGroup> {
|
||||
// Note that sharding works based on test groups.
|
||||
// This means parallel files will be sharded by single tests,
|
||||
// while non-parallel files will be sharded by the whole file.
|
||||
|
|
@ -138,16 +145,85 @@ export function filterForShard(shard: { total: number, current: number }, testGr
|
|||
// Shards are still balanced by the number of tests, not files,
|
||||
// even in the case of non-paralleled files.
|
||||
|
||||
const lengths = new Array(shard.total).fill(0);
|
||||
if (mode === 'round-robin')
|
||||
return filterForShardRoundRobin(shard, testGroups);
|
||||
if (mode === 'duration-round-robin')
|
||||
return filterForShardRoundRobin(shard, testGroups, lastRunInfo);
|
||||
return filterForShardPartition(shard, testGroups);
|
||||
}
|
||||
|
||||
/**
|
||||
* Shards tests by partitioning them into equal parts.
|
||||
*
|
||||
* ```
|
||||
* [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
|
||||
* Shard 1: ^---------^ : [ 1, 2, 3 ]
|
||||
* Shard 2: ^---------^ : [ 4, 5, 6 ]
|
||||
* Shard 3: ^---------^ : [ 7, 8, 9 ]
|
||||
* Shard 4: ^---------^ : [ 10,11,12 ]
|
||||
* ```
|
||||
*/
|
||||
function filterForShardPartition(shard: { total: number, current: number }, testGroups: TestGroup[]): Set<TestGroup> {
|
||||
let shardableTotal = 0;
|
||||
for (const group of testGroups)
|
||||
shardableTotal += group.tests.length;
|
||||
|
||||
// Each shard gets some tests.
|
||||
const shardSize = Math.floor(shardableTotal / shard.total);
|
||||
// First few shards get one more test each.
|
||||
const extraOne = shardableTotal - shardSize * shard.total;
|
||||
|
||||
const currentShard = shard.current - 1; // Make it zero-based for calculations.
|
||||
const from = shardSize * currentShard + Math.min(extraOne, currentShard);
|
||||
const to = from + shardSize + (currentShard < extraOne ? 1 : 0);
|
||||
|
||||
let current = 0;
|
||||
const result = new Set<TestGroup>();
|
||||
for (const group of testGroups) {
|
||||
// Any test group goes to the shard that contains the first test of this group.
|
||||
// So, this shard gets any group that starts at [from; to)
|
||||
if (current >= from && current < to)
|
||||
result.add(group);
|
||||
current += group.tests.length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shards tests by round-robin.
|
||||
*
|
||||
* ```
|
||||
* [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
|
||||
* Shard 1: ^ ^ ^ : [ 1, 5, 9 ]
|
||||
* Shard 2: ^ ^ ^ : [ 2, 6,10 ]
|
||||
* Shard 3: ^ ^ ^ : [ 3, 7,11 ]
|
||||
* Shard 4: ^ ^ ^ : [ 4, 8,12 ]
|
||||
* ```
|
||||
*/
|
||||
function filterForShardRoundRobin(
|
||||
shard: { total: number, current: number },
|
||||
testGroups: TestGroup[],
|
||||
lastRunInfo?: LastRunInfo,
|
||||
): Set<TestGroup> {
|
||||
|
||||
const weights = new Array(shard.total).fill(0);
|
||||
const shardSet = new Array(shard.total).fill(0).map(() => new Set<TestGroup>());
|
||||
const averageDuration = lastRunInfo ? Object.values(lastRunInfo?.testDurations || {}).reduce((a, b) => a + b, 1) / Math.max(1, Object.values(lastRunInfo?.testDurations || {}).length) : 0;
|
||||
const weight = (group: TestGroup) => {
|
||||
if (!lastRunInfo)
|
||||
// If we don't have last run info, we just count the number of tests.
|
||||
return group.tests.length;
|
||||
// If we have last run info, we use the duration of the tests.
|
||||
return group.tests.reduce((sum, test) => sum + Math.max(1, lastRunInfo.testDurations?.[test.id] || averageDuration), 0);
|
||||
};
|
||||
|
||||
// We sort the test groups by the number of tests in descending order.
|
||||
const sortedTestGroups = testGroups.slice().sort((a, b) => b.tests.length - a.tests.length);
|
||||
const sortedTestGroups = testGroups.slice().sort((a, b) => weight(b) - weight(a));
|
||||
|
||||
// Then we add each group to the shard with the smallest number of tests.
|
||||
for (const group of sortedTestGroups) {
|
||||
const index = lengths.reduce((minIndex, currentLength, currentIndex) => currentLength < lengths[minIndex] ? currentIndex : minIndex, 0);
|
||||
lengths[index] += group.tests.length;
|
||||
const index = weights.reduce((minIndex, currentLength, currentIndex) => currentLength < weights[minIndex] ? currentIndex : minIndex, 0);
|
||||
weights[index] += weight(group);
|
||||
shardSet[index].add(group);
|
||||
}
|
||||
|
||||
|
|
|
|||
13
packages/playwright/types/test.d.ts
vendored
13
packages/playwright/types/test.d.ts
vendored
|
|
@ -1425,6 +1425,19 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
|
|||
total: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Defines the algorithm to be used for sharding. Defaults to `'partition'`.
|
||||
* - `'partition'` - divide the set of test groups by number of shards. e.g. first half goes to shard 1/2 and
|
||||
* seconds half to shard 2/2.
|
||||
* - `'round-robin'` - spread test groups to shards in a round-robin way. e.g. loop over test groups and always
|
||||
* assign to the shard that has the lowest number of tests.
|
||||
* - `'duration-round-robin'` - use duration info from `.last-run.json` to spread test groups to shards in a
|
||||
* round-robin way. e.g. loop over test groups and always assign to the shard that has the lowest duration of
|
||||
* tests. new tests which were not present in the last run will use an average duration time. When no
|
||||
* `.last-run.json` could be found the behavior is identical to `'round-robin'`.
|
||||
*/
|
||||
shardingMode?: "partition"|"round-robin"|"duration-round-robin";
|
||||
|
||||
/**
|
||||
* Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is
|
||||
* mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests
|
||||
|
|
|
|||
457
tests/playwright-test/shard-roundrobin.spec.ts
Normal file
457
tests/playwright-test/shard-roundrobin.spec.ts
Normal file
|
|
@ -0,0 +1,457 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { expect, test } from './playwright-test-fixtures';
|
||||
|
||||
/**
|
||||
* Test cases:
|
||||
* - `a1.spec.ts`: 4 tests (default mode)
|
||||
* - `a2.spec.ts`: 2 tests (parallel mode)
|
||||
* - `a3.spec.ts`: 2 tests (parallel mode)
|
||||
* - `a4.spec.ts`: 2 tests (default mode)
|
||||
*
|
||||
* Test Groups distribution for 2 shards
|
||||
* ```
|
||||
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
|
||||
* Shard 1/2: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^5
|
||||
* Shard 2/2: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 ^^^^^^^2 ^^^^^^^2
|
||||
* ```
|
||||
*
|
||||
* Test Groups distribution for 3 shards
|
||||
* ```
|
||||
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
|
||||
* Shard 1/3: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
|
||||
* Shard 2/3: ^^^^^^^5 ^^^^^^^2 ^^^^^^^2
|
||||
* Shard 3/3: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6
|
||||
* ```
|
||||
*
|
||||
* Test Groups distribution for 4 shards
|
||||
* ```
|
||||
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
|
||||
* Shard 1/4: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
|
||||
* Shard 2/4: ^^^^^^^2 ^^^^^^^2
|
||||
* Shard 3/4: ^^^^^^^3 ^^^^^^^5
|
||||
* Shard 4/4: ^^^^^^^4 ^^^^^^^6
|
||||
* ```
|
||||
*
|
||||
* Test Groups distribution for 4 shards with fully-parallel
|
||||
* ```
|
||||
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
|
||||
* Shard 1/4: ^^^^^^^1 ^^^^^^^5 ^^^^^^^9
|
||||
* Shard 2/4: ^^^^^^^2 ^^^^^^^6 ^^^^^^10
|
||||
* Shard 3/4: ^^^^^^^3 ^^^^^^^7 ^^^^^^11
|
||||
* Shard 4/4: ^^^^^^^4 ^^^^^^^8
|
||||
* ```
|
||||
*/
|
||||
const tests = {
|
||||
'a1.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test('test1', async () => {
|
||||
console.log('\\n%%a1-test1-done');
|
||||
});
|
||||
test('test2', async () => {
|
||||
console.log('\\n%%a1-test2-done');
|
||||
});
|
||||
test('test3', async () => {
|
||||
console.log('\\n%%a1-test3-done');
|
||||
});
|
||||
test('test4', async () => {
|
||||
console.log('\\n%%a1-test4-done');
|
||||
});
|
||||
`,
|
||||
'a2.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test.describe.configure({ mode: 'parallel' });
|
||||
test('test1', async () => {
|
||||
console.log('\\n%%a2-test1-done');
|
||||
});
|
||||
test('test2', async () => {
|
||||
console.log('\\n%%a2-test2-done');
|
||||
});
|
||||
`,
|
||||
'a3.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test.describe.configure({ mode: 'parallel' });
|
||||
test('test1', async () => {
|
||||
console.log('\\n%%a3-test1-done');
|
||||
});
|
||||
test('test2', async () => {
|
||||
console.log('\\n%%a3-test2-done');
|
||||
});
|
||||
`,
|
||||
'a4.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test('test1', async () => {
|
||||
console.log('\\n%%a4-test1-done');
|
||||
});
|
||||
test('test2', async () => {
|
||||
console.log('\\n%%a4-test2-done');
|
||||
});
|
||||
`,
|
||||
};
|
||||
|
||||
test('should respect shard=1/2', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/2', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(5);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a1-test1-done',
|
||||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
'a1-test4-done',
|
||||
'a3-test1-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=2/2', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(5);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
'a2-test2-done',
|
||||
'a3-test2-done',
|
||||
'a4-test1-done',
|
||||
'a4-test2-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=1/3', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/3', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(4);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a1-test1-done',
|
||||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
'a1-test4-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=2/3', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/3', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(3);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a3-test1-done',
|
||||
'a4-test1-done',
|
||||
'a4-test2-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=3/3', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '3/3', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(3);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
'a2-test2-done',
|
||||
'a3-test2-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=1/4', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/4', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(4);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a1-test1-done',
|
||||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
'a1-test4-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=2/4', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/4', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a4-test1-done',
|
||||
'a4-test2-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=3/4', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '3/4', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
'a3-test1-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=4/4', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '4/4', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test2-done',
|
||||
'a3-test2-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should not produce skipped tests for zero-sized shards', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '10/10', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(0);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([]);
|
||||
});
|
||||
|
||||
test('should respect shard=1/2 in config', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
...tests,
|
||||
'playwright.config.js': `
|
||||
module.exports = { shardingMode: 'round-robin', shard: { current: 1, total: 2 } };
|
||||
`,
|
||||
}, { workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(5);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a1-test1-done',
|
||||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
'a1-test4-done',
|
||||
'a3-test1-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should work with workers=1 and --fully-parallel', async ({ runInlineTest }) => {
|
||||
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/21226' });
|
||||
const tests = {
|
||||
'a1.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test('should pass', async ({ }) => {
|
||||
});
|
||||
`,
|
||||
'a2.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test('should pass', async ({ }) => {
|
||||
});
|
||||
test.skip('should skip', async ({ }) => {
|
||||
});
|
||||
`,
|
||||
};
|
||||
|
||||
{
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/2', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.skipped).toBe(1);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.skipped).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('should skip dependency when project is sharded out', async ({ runInlineTest }) => {
|
||||
const tests = {
|
||||
'playwright.config.ts': `
|
||||
module.exports = {
|
||||
projects: [
|
||||
{ name: 'setup1', testMatch: /setup.ts/ },
|
||||
{ name: 'tests1', dependencies: ['setup1'] },
|
||||
{ name: 'setup2', testMatch: /setup.ts/ },
|
||||
{ name: 'tests2', dependencies: ['setup2'] },
|
||||
],
|
||||
};
|
||||
`,
|
||||
'test.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test('test', async ({}) => {
|
||||
console.log('\\n%%test in ' + test.info().project.name);
|
||||
});
|
||||
`,
|
||||
'setup.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test('setup', async ({}) => {
|
||||
console.log('\\n%%setup in ' + test.info().project.name);
|
||||
});
|
||||
`,
|
||||
};
|
||||
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'setup in setup2',
|
||||
'test in tests2',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should not shard mode:default suites', async ({ runInlineTest }) => {
|
||||
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/22891' });
|
||||
|
||||
const tests = {
|
||||
'a1.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test('test0', async ({ }) => {
|
||||
console.log('\\n%%test0');
|
||||
});
|
||||
test('test1', async ({ }) => {
|
||||
console.log('\\n%%test1');
|
||||
});
|
||||
`,
|
||||
'a2.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test.describe.configure({ mode: 'parallel' });
|
||||
|
||||
test.describe(() => {
|
||||
test.describe.configure({ mode: 'default' });
|
||||
test.beforeAll(() => {
|
||||
console.log('\\n%%beforeAll1');
|
||||
});
|
||||
test('test2', async ({ }) => {
|
||||
console.log('\\n%%test2');
|
||||
});
|
||||
test('test3', async ({ }) => {
|
||||
console.log('\\n%%test3');
|
||||
});
|
||||
});
|
||||
|
||||
test.describe(() => {
|
||||
test.describe.configure({ mode: 'default' });
|
||||
test.beforeAll(() => {
|
||||
console.log('\\n%%beforeAll2');
|
||||
});
|
||||
test('test4', async ({ }) => {
|
||||
console.log('\\n%%test4');
|
||||
});
|
||||
test('test5', async ({ }) => {
|
||||
console.log('\\n%%test5');
|
||||
});
|
||||
});
|
||||
`,
|
||||
};
|
||||
|
||||
{
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/3', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.outputLines).toEqual(['beforeAll1', 'test2', 'test3']);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '3/3', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']);
|
||||
}
|
||||
});
|
||||
|
||||
test('should not shard mode:serial suites when fully-parallel', async ({ runInlineTest }) => {
|
||||
const tests = {
|
||||
'a1.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test.describe.configure({ mode: 'serial' });
|
||||
test('test1', async ({ }) => {
|
||||
console.log('\\n%%a1-test1-done');
|
||||
});
|
||||
test('test2', async ({ }) => {
|
||||
console.log('\\n%%a1-test2-done');
|
||||
});
|
||||
test('test3', async ({ }) => {
|
||||
console.log('\\n%%a1-test3-done');
|
||||
});
|
||||
`,
|
||||
'a2.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test.describe(() => {
|
||||
test('test1', async ({ }) => {
|
||||
console.log('\\n%%a2-test1-done');
|
||||
});
|
||||
test('test2', async ({ }) => {
|
||||
console.log('\\n%%a2-test2-done');
|
||||
});
|
||||
});
|
||||
`,
|
||||
'a3.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test.describe(() => {
|
||||
test('test1', async ({ }) => {
|
||||
console.log('\\n%%a3-test1-done');
|
||||
});
|
||||
test('test2', async ({ }) => {
|
||||
console.log('\\n%%a3-test2-done');
|
||||
});
|
||||
});
|
||||
`,
|
||||
};
|
||||
|
||||
{
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/2', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(4);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a1-test1-done',
|
||||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
'a3-test2-done',
|
||||
]);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/2', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(3);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
'a2-test2-done',
|
||||
'a3-test1-done',
|
||||
]);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '1/5', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(3);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a1-test1-done',
|
||||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
]);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '2/5', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
]);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { ['sharding-mode']: 'round-robin', shard: '5/5', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a3-test2-done',
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
|
@ -14,48 +14,8 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { expect, test } from './playwright-test-fixtures';
|
||||
import { test, expect } from './playwright-test-fixtures';
|
||||
|
||||
/**
|
||||
* Test cases:
|
||||
* - `a1.spec.ts`: 4 tests (default mode)
|
||||
* - `a2.spec.ts`: 2 tests (parallel mode)
|
||||
* - `a3.spec.ts`: 2 tests (parallel mode)
|
||||
* - `a4.spec.ts`: 2 tests (default mode)
|
||||
*
|
||||
* Test Groups distribution for 2 shards
|
||||
* ```
|
||||
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
|
||||
* Shard 1/2: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^5
|
||||
* Shard 2/2: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 ^^^^^^^2 ^^^^^^^2
|
||||
* ```
|
||||
*
|
||||
* Test Groups distribution for 3 shards
|
||||
* ```
|
||||
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
|
||||
* Shard 1/3: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
|
||||
* Shard 2/3: ^^^^^^^5 ^^^^^^^2 ^^^^^^^2
|
||||
* Shard 3/3: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6
|
||||
* ```
|
||||
*
|
||||
* Test Groups distribution for 4 shards
|
||||
* ```
|
||||
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
|
||||
* Shard 1/4: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
|
||||
* Shard 2/4: ^^^^^^^2 ^^^^^^^2
|
||||
* Shard 3/4: ^^^^^^^3 ^^^^^^^5
|
||||
* Shard 4/4: ^^^^^^^4 ^^^^^^^6
|
||||
* ```
|
||||
*
|
||||
* Test Groups distribution for 4 shards with fully-parallel
|
||||
* ```
|
||||
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
|
||||
* Shard 1/4: ^^^^^^^1 ^^^^^^^5 ^^^^^^^9
|
||||
* Shard 2/4: ^^^^^^^2 ^^^^^^^6 ^^^^^^10
|
||||
* Shard 3/4: ^^^^^^^3 ^^^^^^^7 ^^^^^^11
|
||||
* Shard 4/4: ^^^^^^^4 ^^^^^^^8
|
||||
* ```
|
||||
*/
|
||||
const tests = {
|
||||
'a1.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
|
|
@ -113,7 +73,7 @@ test('should respect shard=1/2', async ({ runInlineTest }) => {
|
|||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
'a1-test4-done',
|
||||
'a3-test1-done',
|
||||
'a2-test1-done',
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
@ -123,8 +83,8 @@ test('should respect shard=2/2', async ({ runInlineTest }) => {
|
|||
expect(result.passed).toBe(5);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
'a2-test2-done',
|
||||
'a3-test1-done',
|
||||
'a3-test2-done',
|
||||
'a4-test1-done',
|
||||
'a4-test2-done',
|
||||
|
|
@ -150,9 +110,9 @@ test('should respect shard=2/3', async ({ runInlineTest }) => {
|
|||
expect(result.passed).toBe(3);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
'a2-test2-done',
|
||||
'a3-test1-done',
|
||||
'a4-test1-done',
|
||||
'a4-test2-done',
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
@ -162,31 +122,7 @@ test('should respect shard=3/3', async ({ runInlineTest }) => {
|
|||
expect(result.passed).toBe(3);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
'a2-test2-done',
|
||||
'a3-test2-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=1/4', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { shard: '1/4', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(4);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a1-test1-done',
|
||||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
'a1-test4-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=2/4', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { shard: '2/4', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a4-test1-done',
|
||||
'a4-test2-done',
|
||||
]);
|
||||
|
|
@ -198,18 +134,7 @@ test('should respect shard=3/4', async ({ runInlineTest }) => {
|
|||
expect(result.passed).toBe(2);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
'a3-test1-done',
|
||||
]);
|
||||
});
|
||||
|
||||
test('should respect shard=4/4', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest(tests, { shard: '4/4', workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.skipped).toBe(0);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test2-done',
|
||||
'a3-test2-done',
|
||||
]);
|
||||
});
|
||||
|
|
@ -238,7 +163,7 @@ test('should respect shard=1/2 in config', async ({ runInlineTest }) => {
|
|||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
'a1-test4-done',
|
||||
'a3-test1-done',
|
||||
'a2-test1-done',
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
@ -249,28 +174,20 @@ test('should work with workers=1 and --fully-parallel', async ({ runInlineTest }
|
|||
import { test } from '@playwright/test';
|
||||
test('should pass', async ({ }) => {
|
||||
});
|
||||
test.skip('should skip', async ({ }) => {
|
||||
});
|
||||
`,
|
||||
'a2.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test('should pass', async ({ }) => {
|
||||
});
|
||||
test.skip('should skip', async ({ }) => {
|
||||
});
|
||||
`,
|
||||
};
|
||||
|
||||
{
|
||||
const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.skipped).toBe(1);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.skipped).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('should skip dependency when project is sharded out', async ({ runInlineTest }) => {
|
||||
|
|
@ -367,91 +284,3 @@ test('should not shard mode:default suites', async ({ runInlineTest }) => {
|
|||
expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']);
|
||||
}
|
||||
});
|
||||
|
||||
test('should not shard mode:serial suites when fully-parallel', async ({ runInlineTest }) => {
|
||||
const tests = {
|
||||
'a1.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test.describe.configure({ mode: 'serial' });
|
||||
test('test1', async ({ }) => {
|
||||
console.log('\\n%%a1-test1-done');
|
||||
});
|
||||
test('test2', async ({ }) => {
|
||||
console.log('\\n%%a1-test2-done');
|
||||
});
|
||||
test('test3', async ({ }) => {
|
||||
console.log('\\n%%a1-test3-done');
|
||||
});
|
||||
`,
|
||||
'a2.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test.describe(() => {
|
||||
test('test1', async ({ }) => {
|
||||
console.log('\\n%%a2-test1-done');
|
||||
});
|
||||
test('test2', async ({ }) => {
|
||||
console.log('\\n%%a2-test2-done');
|
||||
});
|
||||
});
|
||||
`,
|
||||
'a3.spec.ts': `
|
||||
import { test } from '@playwright/test';
|
||||
test.describe(() => {
|
||||
test('test1', async ({ }) => {
|
||||
console.log('\\n%%a3-test1-done');
|
||||
});
|
||||
test('test2', async ({ }) => {
|
||||
console.log('\\n%%a3-test2-done');
|
||||
});
|
||||
});
|
||||
`,
|
||||
};
|
||||
|
||||
{
|
||||
const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(4);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a1-test1-done',
|
||||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
'a3-test2-done',
|
||||
]);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(3);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
'a2-test2-done',
|
||||
'a3-test1-done',
|
||||
]);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { shard: '1/5', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(3);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a1-test1-done',
|
||||
'a1-test2-done',
|
||||
'a1-test3-done',
|
||||
]);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { shard: '2/5', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a2-test1-done',
|
||||
]);
|
||||
}
|
||||
{
|
||||
const result = await runInlineTest(tests, { shard: '5/5', ['fully-parallel']: true, workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.passed).toBe(1);
|
||||
expect(result.outputLines).toEqual([
|
||||
'a3-test2-done',
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue