From 4f1027bdd0bdaf40a31ed63bd31e3a9e3166fa36 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Mon, 1 Nov 2021 10:37:34 -0700 Subject: [PATCH] feat(test runner): introduce TestInfo.parallelIndex (#9762) This is a worker number between `0` and `workers - 1` that does not change after worker process restart. --- docs/src/test-api/class-testinfo.md | 11 ++++- docs/src/test-api/class-workerinfo.md | 12 +++++- docs/src/test-parallel-js.md | 6 ++- packages/playwright-test/src/dispatcher.ts | 12 ++++-- packages/playwright-test/src/ipc.ts | 1 + packages/playwright-test/src/workerRunner.ts | 2 + packages/playwright-test/types/test.d.ts | 32 ++++++++++++-- tests/playwright-test/worker-index.spec.ts | 44 +++++++++++++++++++- utils/generate_types/overrides-test.d.ts | 2 + 9 files changed, 108 insertions(+), 14 deletions(-) diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index c8ca2e6ad3..75ea2162fe 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -201,6 +201,13 @@ test('example test', async ({}, testInfo) => { Path segments to append at the end of the resulting path. +## property: TestInfo.parallelIndex +- type: <[int]> + +The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have a different `parallelIndex`. When a worker is restarted, for example after a failure, the new worker process has the same `parallelIndex`. + +Also available as `process.env.TEST_PARALLEL_INDEX`. Learn more about [parallelism and sharding](./test-parallel.md) with Playwright Test. + ## property: TestInfo.project - type: <[TestProject]> @@ -358,4 +365,6 @@ The title of the currently running test as passed to `test(title, testFunction)` ## property: TestInfo.workerIndex - type: <[int]> -The unique index of the worker process that is running the test. Also available as `process.env.TEST_WORKER_INDEX`. Learn more about [parallelism and sharding](./test-parallel.md) with Playwright Test. +The unique index of the worker process that is running the test. When a worker is restarted, for example after a failure, the new worker process gets a new unique `workerIndex`. + +Also available as `process.env.TEST_WORKER_INDEX`. Learn more about [parallelism and sharding](./test-parallel.md) with Playwright Test. diff --git a/docs/src/test-api/class-workerinfo.md b/docs/src/test-api/class-workerinfo.md index 197b1ea0b6..d11ea6afef 100644 --- a/docs/src/test-api/class-workerinfo.md +++ b/docs/src/test-api/class-workerinfo.md @@ -25,6 +25,14 @@ test.beforeAll(async ({ browserName }, workerInfo) => { Processed configuration from the [configuration file](./test-configuration.md). +## property: WorkerInfo.parallelIndex +- type: <[int]> + +The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have a different `parallelIndex`. When a worker is restarted, for example after a failure, the new worker process has the same `parallelIndex`. + +Also available as `process.env.TEST_PARALLEL_INDEX`. Learn more about [parallelism and sharding](./test-parallel.md) with Playwright Test. + + ## property: WorkerInfo.project - type: <[TestProject]> @@ -34,4 +42,6 @@ Processed project configuration from the [configuration file](./test-configurati ## property: WorkerInfo.workerIndex - type: <[int]> -The unique index of the worker process that is running the test. Also available as `process.env.TEST_WORKER_INDEX`. Learn more about [parallelism and sharding](./test-parallel.md) with Playwright Test. +The unique index of the worker process that is running the test. When a worker is restarted, for example after a failure, the new worker process gets a new unique `workerIndex`. + +Also available as `process.env.TEST_WORKER_INDEX`. Learn more about [parallelism and sharding](./test-parallel.md) with Playwright Test. diff --git a/docs/src/test-parallel-js.md b/docs/src/test-parallel-js.md index dc115dd58a..f53b9916b1 100644 --- a/docs/src/test-parallel-js.md +++ b/docs/src/test-parallel-js.md @@ -135,6 +135,8 @@ const config: PlaywrightTestConfig = { export default config; ``` -## Worker index +## Worker index and parallel index -Each worker process is assigned a unique id (an index that starts with 1). You can read it from environment variable `process.env.TEST_WORKER_INDEX`, or access through [`property: TestInfo.workerIndex`]. +Each worker process is assigned two ids: a unique worker index that starts with 1, and a parallel index that is between `0` and `workers - 1`. When a worker is restarted, for example after a failure, the new worker process has the same `parallelIndex` and a new `workerIndex`. + +You can read an index from environment variables `process.env.TEST_WORKER_INDEX` and `process.env.TEST_PARALLEL_INDEX`, or access them through [`property: TestInfo.workerIndex`] and [`property: TestInfo.parallelIndex`]. diff --git a/packages/playwright-test/src/dispatcher.ts b/packages/playwright-test/src/dispatcher.ts index b831ca0c03..7a8f024b22 100644 --- a/packages/playwright-test/src/dispatcher.ts +++ b/packages/playwright-test/src/dispatcher.ts @@ -96,7 +96,7 @@ export class Dispatcher { // 2. Start the worker if it is down. if (!worker) { - worker = this._createWorker(job.workerHash); + worker = this._createWorker(job.workerHash, index); this._workerSlots[index].worker = worker; worker.on('exit', () => this._workerSlots[index].worker = undefined); await worker.init(job, this._loader.serialize()); @@ -337,8 +337,8 @@ export class Dispatcher { return result; } - _createWorker(hash: string) { - const worker = new Worker(hash); + _createWorker(hash: string, parallelIndex: number) { + const worker = new Worker(hash, parallelIndex); worker.on('stdOut', (params: TestOutputPayload) => { const chunk = chunkFromParams(params); if (worker.didFail()) { @@ -404,15 +404,17 @@ let lastWorkerIndex = 0; class Worker extends EventEmitter { private process: child_process.ChildProcess; private _hash: string; + private parallelIndex: number; private workerIndex: number; private _didSendStop = false; private _didFail = false; private didExit = false; - constructor(hash: string) { + constructor(hash: string, parallelIndex: number) { super(); this.workerIndex = lastWorkerIndex++; this._hash = hash; + this.parallelIndex = parallelIndex; this.process = child_process.fork(path.join(__dirname, 'worker.js'), { detached: false, @@ -420,6 +422,7 @@ class Worker extends EventEmitter { FORCE_COLOR: process.stdout.isTTY ? '1' : '0', DEBUG_COLORS: process.stdout.isTTY ? '1' : '0', TEST_WORKER_INDEX: String(this.workerIndex), + TEST_PARALLEL_INDEX: String(this.parallelIndex), ...process.env }, // Can't pipe since piping slows down termination for some reason. @@ -439,6 +442,7 @@ class Worker extends EventEmitter { async init(testGroup: TestGroup, loaderData: SerializedLoaderData) { const params: WorkerInitParams = { workerIndex: this.workerIndex, + parallelIndex: this.parallelIndex, repeatEachIndex: testGroup.repeatEachIndex, projectIndex: testGroup.projectIndex, loader: loaderData, diff --git a/packages/playwright-test/src/ipc.ts b/packages/playwright-test/src/ipc.ts index 41d9d65aef..b84e2a1e15 100644 --- a/packages/playwright-test/src/ipc.ts +++ b/packages/playwright-test/src/ipc.ts @@ -24,6 +24,7 @@ export type SerializedLoaderData = { }; export type WorkerInitParams = { workerIndex: number; + parallelIndex: number; repeatEachIndex: number; projectIndex: number; loader: SerializedLoaderData; diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index 518ee32997..c6361e0396 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -135,6 +135,7 @@ export class WorkerRunner extends EventEmitter { this._workerInfo = { workerIndex: this._params.workerIndex, + parallelIndex: this._params.parallelIndex, project: this._project.config, config: this._loader.fullConfig(), }; @@ -244,6 +245,7 @@ export class WorkerRunner extends EventEmitter { let lastStepId = 0; const testInfo: TestInfoImpl = { workerIndex: this._params.workerIndex, + parallelIndex: this._params.parallelIndex, project: this._project.config, config: this._loader.fullConfig(), title: test.title, diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 7aa848c008..a8cdbe3425 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -1030,13 +1030,25 @@ export interface WorkerInfo { * Processed configuration from the [configuration file](https://playwright.dev/docs/test-configuration). */ config: FullConfig; + /** + * The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have a + * different `parallelIndex`. When a worker is restarted, for example after a failure, the new worker process has the same + * `parallelIndex`. + * + * Also available as `process.env.TEST_PARALLEL_INDEX`. Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) + * with Playwright Test. + */ + parallelIndex: number; /** * Processed project configuration from the [configuration file](https://playwright.dev/docs/test-configuration). */ project: FullProject; /** - * The unique index of the worker process that is running the test. Also available as `process.env.TEST_WORKER_INDEX`. - * Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with Playwright Test. + * The unique index of the worker process that is running the test. When a worker is restarted, for example after a + * failure, the new worker process gets a new unique `workerIndex`. + * + * Also available as `process.env.TEST_WORKER_INDEX`. Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with + * Playwright Test. */ workerIndex: number; } @@ -1063,13 +1075,25 @@ export interface TestInfo { * Processed configuration from the [configuration file](https://playwright.dev/docs/test-configuration). */ config: FullConfig; + /** + * The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have a + * different `parallelIndex`. When a worker is restarted, for example after a failure, the new worker process has the same + * `parallelIndex`. + * + * Also available as `process.env.TEST_PARALLEL_INDEX`. Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) + * with Playwright Test. + */ + parallelIndex: number; /** * Processed project configuration from the [configuration file](https://playwright.dev/docs/test-configuration). */ project: FullProject; /** - * The unique index of the worker process that is running the test. Also available as `process.env.TEST_WORKER_INDEX`. - * Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with Playwright Test. + * The unique index of the worker process that is running the test. When a worker is restarted, for example after a + * failure, the new worker process gets a new unique `workerIndex`. + * + * Also available as `process.env.TEST_WORKER_INDEX`. Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with + * Playwright Test. */ workerIndex: number; diff --git a/tests/playwright-test/worker-index.spec.ts b/tests/playwright-test/worker-index.spec.ts index 8f56bfca49..62c5c2d5d0 100644 --- a/tests/playwright-test/worker-index.spec.ts +++ b/tests/playwright-test/worker-index.spec.ts @@ -24,6 +24,7 @@ test('should run in parallel', async ({ runInlineTest }) => { const { test } = pwt; test('succeeds', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); // First test waits for the second to start to work around the race. while (true) { if (fs.existsSync(path.join(testInfo.project.outputDir, 'parallel-index.txt'))) @@ -41,6 +42,7 @@ test('should run in parallel', async ({ runInlineTest }) => { fs.mkdirSync(testInfo.project.outputDir, { recursive: true }); fs.writeFileSync(path.join(testInfo.project.outputDir, 'parallel-index.txt'), 'TRUE'); expect(testInfo.workerIndex).toBe(1); + expect(testInfo.parallelIndex).toBe(1); }); `, }); @@ -54,14 +56,17 @@ test('should reuse worker for multiple tests', async ({ runInlineTest }) => { const { test } = pwt; test('succeeds 1', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); test('succeeds 2', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); test('succeeds 3', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); `, }); @@ -75,15 +80,18 @@ test('should reuse worker after test.fixme()', async ({ runInlineTest }) => { const { test } = pwt; test('succeeds 1', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); test('fixme 1', async ({}, testInfo) => { test.fixme(); expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); test('succeeds 2', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); `, }); @@ -98,15 +106,18 @@ test('should reuse worker after test.skip()', async ({ runInlineTest }) => { const { test } = pwt; test('succeeds 1', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); test('skip 1', async ({}, testInfo) => { test.skip(); expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); test('succeeds 2', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); `, }); @@ -121,6 +132,7 @@ test('should not use new worker after test.fail()', async ({ runInlineTest }) => const { test } = pwt; test('succeeds 1', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); test('fail 1', async ({}, testInfo) => { @@ -130,6 +142,7 @@ test('should not use new worker after test.fail()', async ({ runInlineTest }) => test('succeeds 2', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); `, }); @@ -144,6 +157,7 @@ test('should use new worker after test failure', async ({ runInlineTest }) => { const { test } = pwt; test('succeeds 1', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(0); + expect(testInfo.parallelIndex).toBe(0); }); test('fail 1', async ({}, testInfo) => { @@ -152,9 +166,10 @@ test('should use new worker after test failure', async ({ runInlineTest }) => { test('succeeds 2', async ({}, testInfo) => { expect(testInfo.workerIndex).toBe(1); + expect(testInfo.parallelIndex).toBe(0); }); `, - }); + }, { workers: 1 }); expect(result.passed).toBe(2); expect(result.failed).toBe(1); expect(result.exitCode).toBe(1); @@ -169,13 +184,38 @@ test('should not reuse worker for different suites', async ({ runInlineTest }) = const { test } = pwt; test('succeeds', async ({}, testInfo) => { console.log('workerIndex-' + testInfo.workerIndex); + console.log('parallelIndex-' + testInfo.parallelIndex); }); `, - }); + }, { workers: 1 }); expect(result.passed).toBe(3); expect(result.exitCode).toBe(0); expect(result.results.map(r => r.workerIndex).sort()).toEqual([0, 1, 2]); expect(result.output).toContain('workerIndex-0'); expect(result.output).toContain('workerIndex-1'); expect(result.output).toContain('workerIndex-2'); + expect(result.output).toContain('parallelIndex-0'); + expect(result.output).not.toContain('parallelIndex-1'); +}); + +test('parallelIndex should be in 0..workers-1', async ({ runInlineTest }) => { + const files = {}; + for (let i = 0; i < 10; i++) { + files[`a${i}.test.js`] = ` + const { test } = pwt; + test('passes-1', async ({}, testInfo) => { + await new Promise(f => setTimeout(f, 100 + 50 * ${i})); + expect(testInfo.parallelIndex >= 0).toBeTruthy(); + expect(testInfo.parallelIndex < testInfo.config.workers).toBeTruthy(); + }); + test('passes-2', async ({}, testInfo) => { + await new Promise(f => setTimeout(f, 100 + 50 * ${i})); + expect(testInfo.parallelIndex >= 0).toBeTruthy(); + expect(testInfo.parallelIndex < testInfo.config.workers).toBeTruthy(); + }); + `; + } + const result = await runInlineTest(files, { workers: 3 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(20); }); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 52ea4bd6b8..b426a16a22 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -166,12 +166,14 @@ export interface TestError { export interface WorkerInfo { config: FullConfig; + parallelIndex: number; project: FullProject; workerIndex: number; } export interface TestInfo { config: FullConfig; + parallelIndex: number; project: FullProject; workerIndex: number;