From d4e236e27e3bb785c5107abac9a89668e343c086 Mon Sep 17 00:00:00 2001 From: Mathias Leppich Date: Wed, 15 May 2024 09:28:20 +0200 Subject: [PATCH] feat(test runner): shuffle order of tests with sharding seed --- docs/src/test-api/class-testconfig.md | 21 ++++++++ docs/src/test-sharding-js.md | 6 +++ packages/playwright/src/common/config.ts | 1 + packages/playwright/src/common/ipc.ts | 1 + packages/playwright/src/program.ts | 2 + packages/playwright/src/runner/loadUtils.ts | 4 ++ packages/playwright/src/runner/shuffle.ts | 59 +++++++++++++++++++++ packages/playwright/types/test.d.ts | 12 +++++ 8 files changed, 106 insertions(+) create mode 100644 packages/playwright/src/runner/shuffle.ts diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 2d7329a027..02b50bcdc4 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -476,6 +476,27 @@ export default defineConfig({ ``` +## property: TestConfig.shardingSeed + +* since: ??? +- type: ?<[string]> + +Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests in a deterministic way can help to distribute the load more evenly. + +The sharding seed is a string that is used to initialize a random number generator. The configured value needs to be the same for all shards, to make sure + +Learn more about [parallelism and sharding](../test-parallel.md) with Playwright Test. + +**Usage** + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + shardingSeed: "string value" +}); +``` + ## property: TestConfig.testDir * since: v1.10 - type: ?<[string]> diff --git a/docs/src/test-sharding-js.md b/docs/src/test-sharding-js.md index 068b7172d0..f5fe176a87 100644 --- a/docs/src/test-sharding-js.md +++ b/docs/src/test-sharding-js.md @@ -22,6 +22,12 @@ Now, if you run these shards in parallel on different computers, your test suite Note that Playwright can only shard tests that can be run in parallel. By default, this means Playwright will shard test files. Learn about other options in the [parallelism guide](./test-parallel.md). +## Randomizing test order in a deterministic way + +By default tests are run in the order they are discovered, which is mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. For example, if the first half of your tests are slower than the rest of your tests and you are using 4 shards it means that shard 1 and 2 will take significantly more time then shard 3 and 4. + +To aid with this problem you can pass `--sharding-seed=string-value` to randomize the order of tests in a deterministic way, which could yield better distribution of slow and fast tests across all shards. + ## Merging reports from multiple shards In the previous example, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them. diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 7bc56cf4f6..22d858e368 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -86,6 +86,7 @@ export class FullConfigInternal { quiet: takeFirst(configCLIOverrides.quiet, userConfig.quiet, false), projects: [], shard: takeFirst(configCLIOverrides.shard, userConfig.shard, null), + shardingSeed: takeFirst(configCLIOverrides.shardingSeed, userConfig.shardingSeed, undefined), updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'), version: require('../../package.json').version, workers: 0, diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index c1c7b2da25..f5e3ec0858 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -32,6 +32,7 @@ export type ConfigCLIOverrides = { reporter?: ReporterDescription[]; additionalReporters?: ReporterDescription[]; shard?: { current: number, total: number }; + shardingSeed?: string; timeout?: number; ignoreSnapshots?: boolean; updateSnapshots?: 'all'|'none'|'missing'; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index feca8d644b..19b0bf9969 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -279,6 +279,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid retries: options.retries ? parseInt(options.retries, 10) : undefined, reporter: resolveReporterOption(options.reporter), shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined, + shardingSeed: options.shardingSeed ? options.shardingSeed : undefined, timeout: options.timeout ? parseInt(options.timeout, 10) : undefined, ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined, updateSnapshots: options.updateSnapshots ? 'all' as const : undefined, @@ -355,6 +356,7 @@ const testOptions: [string, string][] = [ ['--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`], ['--retries ', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`], ['--shard ', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`], + ['--sharding-seed ', `Seed string for randomizing the test order before sharding. Defaults to not randomizing the order.`], ['--timeout ', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`], ['--trace ', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`], ['--ui', `Run tests in interactive UI mode`], diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index b138b6fae4..bbcdc15aa1 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -31,6 +31,7 @@ import { createTestGroups, filterForShard, type TestGroup } from './testGroups'; import { dependenciesForTestFile } from '../transform/compilationCache'; import { sourceMapSupport } from '../utilsBundle'; import type { RawSourceMap } from 'source-map'; +import { shuffleWithSeed } from './shuffle'; export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean, additionalFileMatcher?: Matcher) { const config = testRun.config; @@ -181,6 +182,9 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho for (const projectSuite of rootSuite.suites) testGroups.push(...createTestGroups(projectSuite, config.config.workers)); + if (config.config.shardingSeed) + shuffleWithSeed(testGroups, config.config.shardingSeed); + // Shard test groups. const testGroupsInThisShard = filterForShard(config.config.shard, testGroups); const testsInThisShard = new Set(); diff --git a/packages/playwright/src/runner/shuffle.ts b/packages/playwright/src/runner/shuffle.ts new file mode 100644 index 0000000000..32f8498a1d --- /dev/null +++ b/packages/playwright/src/runner/shuffle.ts @@ -0,0 +1,59 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shuffles the given array of items using the given seed. + * + * @param items The array of items to shuffle. + * @param seed The seed to use for shuffling. + */ +export function shuffleWithSeed(items: any[], seed: string): void { + const random = rng(cyrb32(seed)); + for (let i = items.length - 1; i > 0; i--) { + const j = Math.floor(random() * (i + 1)); + [items[i], items[j]] = [items[j], items[i]]; + } +} + +/** + * Returns a random number generator seeded with the given seed. + * + * @param seed The seed for the random number generator. + * @returns The random number generator. + */ +function rng(seed: number) { + const m = 2 ** 35 - 31; + const a = 185852; + let s = seed % m; + return function() { + return (s = s * a % m) / m; + }; +} + +/** + * Return a 32-bit hash from a string. + * + * @param str The string to hash. + * @returns The 32-bit hash. + */ +function cyrb32(str: string) { + let h = 0x2323; + for (let i = 0; i < str.length; i++) { + h = h ^ str.charCodeAt(i); + h = Math.imul(h, 2654435761); + } + return h >>> 0; +} diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 9e050e2901..bdb12aff56 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1407,6 +1407,13 @@ interface TestConfig { total: number; }; + /** + * Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is + * mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests + * in a deterministic way can help to distribute the load more evenly. + */ + shardingSeed?: string + /** * **NOTE** Use * [testConfig.snapshotPathTemplate](https://playwright.dev/docs/api/class-testconfig#test-config-snapshot-path-template) @@ -1783,6 +1790,11 @@ export interface FullConfig { current: number; }; + /** + * See [testConfig.shardingSeed](https://playwright.dev/docs/api/class-testconfig#test-config-sharding-seed). + */ + shardingSeed: string; + /** * See [testConfig.updateSnapshots](https://playwright.dev/docs/api/class-testconfig#test-config-update-snapshots). */