From 423a0aae31e3551b84fb8dc82c8bdc38f6d96ac8 Mon Sep 17 00:00:00 2001 From: Christopher Tangonan <161169629+cTangonan123@users.noreply.github.com> Date: Wed, 12 Feb 2025 02:36:32 +0000 Subject: [PATCH 1/5] feat(parallelIndex): added parallelIndex to JSONReportTestResult Co-authored-by: Neel Deshmukh Co-authored-by: Marcelo Villalobos Diaz --- packages/playwright/src/reporters/json.ts | 1 + packages/playwright/types/testReporter.d.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts index db40750d36..d6833dba02 100644 --- a/packages/playwright/src/reporters/json.ts +++ b/packages/playwright/src/reporters/json.ts @@ -203,6 +203,7 @@ class JSONReporter implements ReporterV2 { const steps = result.steps.filter(s => s.category === 'test.step'); const jsonResult: JSONReportTestResult = { workerIndex: result.workerIndex, + parallelIndex: result.parallelIndex, status: result.status, duration: result.duration, error: result.error, diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 9662faee73..2abfa9592e 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -291,6 +291,7 @@ export interface JSONReportError { export interface JSONReportTestResult { workerIndex: number; + parallelIndex: number; status: TestStatus | undefined; duration: number; error: TestError | undefined; From b9efcc183bb4a748cee3e6570fa506a7e8a2a9a1 Mon Sep 17 00:00:00 2001 From: Christopher Tangonan <161169629+cTangonan123@users.noreply.github.com> Date: Sat, 15 Feb 2025 00:56:37 +0000 Subject: [PATCH 2/5] feat(json-reporter): added parallelIndex to overrides-testReporter.d.ts and added initial test to reporter-json.spec.ts Co-authored-by: Neel Deshmukh Co-authored-by: Marcelo Villalobos Diaz --- tests/playwright-test/reporter-json.spec.ts | 26 +++++++++++++++++++ .../overrides-testReporter.d.ts | 1 + 2 files changed, 27 insertions(+) diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts index f8ec8b9b0b..b08cb10199 100644 --- a/tests/playwright-test/reporter-json.spec.ts +++ b/tests/playwright-test/reporter-json.spec.ts @@ -327,3 +327,29 @@ test.describe('report location', () => { expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true); }); }); + +test('should report parallelIndex', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'tests/a.spec.js': ` + import { test, expect } from '@playwright/test'; + const fs = require('fs'); + test('test 1 passes!', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('test 2 fails!', async ({}) => { + expect(1 + 1).toBe(3); + }); + test('test 3 passes!', async ({}) => { + expect(1 + 1).toBe(2); + }); + ` + }); + expect(result.passed).toBe(2); + expect(result.failed).toBe(1); + expect(result.results[0].workerIndex).toBe(0); + expect(result.results[0].parallelIndex).toBe(0); + expect(result.results[1].workerIndex).toBe(0); + expect(result.results[1].parallelIndex).toBe(0); + expect(result.results[2].workerIndex).toBe(1); + expect(result.results[2].parallelIndex).toBe(0); +}); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index b2e8f0e9e6..70365a911e 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -109,6 +109,7 @@ export interface JSONReportError { export interface JSONReportTestResult { workerIndex: number; + parallelIndex: number; status: TestStatus | undefined; duration: number; error: TestError | undefined; From 945e5a7848f69382c4dee714914acfdd61d52144 Mon Sep 17 00:00:00 2001 From: Christopher Tangonan <161169629+cTangonan123@users.noreply.github.com> Date: Thu, 20 Feb 2025 22:53:36 +0000 Subject: [PATCH 3/5] fix(reporter-json): added config specs for parallelIndex testing Co-authored-by: Neel Deshmukh Co-authored-by: Marcelo Villalobos Diaz --- tests/playwright-test/reporter-json.spec.ts | 22 +++++++++------------ 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts index b08cb10199..79eb3c8296 100644 --- a/tests/playwright-test/reporter-json.spec.ts +++ b/tests/playwright-test/reporter-json.spec.ts @@ -333,23 +333,19 @@ test('should report parallelIndex', async ({ runInlineTest }, testInfo) => { 'tests/a.spec.js': ` import { test, expect } from '@playwright/test'; const fs = require('fs'); - test('test 1 passes!', async ({}) => { - expect(1 + 1).toBe(2); - }); + test('test 1 passes!', async ({}) => {}); test('test 2 fails!', async ({}) => { expect(1 + 1).toBe(3); }); - test('test 3 passes!', async ({}) => { - expect(1 + 1).toBe(2); - }); + test('test 3 passes!', async ({}) => {}); ` - }); + }, { 'workers': '1', 'reporter': 'json' }); expect(result.passed).toBe(2); expect(result.failed).toBe(1); - expect(result.results[0].workerIndex).toBe(0); - expect(result.results[0].parallelIndex).toBe(0); - expect(result.results[1].workerIndex).toBe(0); - expect(result.results[1].parallelIndex).toBe(0); - expect(result.results[2].workerIndex).toBe(1); - expect(result.results[2].parallelIndex).toBe(0); + expect(result.report.suites[0].specs[0].tests[0].results[0].workerIndex).toBe(0); + expect(result.report.suites[0].specs[1].tests[0].results[0].workerIndex).toBe(0); + expect(result.report.suites[0].specs[2].tests[0].results[0].workerIndex).toBe(1); + expect(result.report.suites[0].specs[0].tests[0].results[0].parallelIndex).toBe(0); + expect(result.report.suites[0].specs[1].tests[0].results[0].parallelIndex).toBe(0); + expect(result.report.suites[0].specs[2].tests[0].results[0].parallelIndex).toBe(0); }); From 10d10d49b51d01d601a23afdda14a89f1dd7e6a6 Mon Sep 17 00:00:00 2001 From: cTangonan123 Date: Tue, 25 Feb 2025 23:39:13 -0800 Subject: [PATCH 4/5] feat(json-reporter): configured test for parallelIndex to be ran in parallel Co-authored-by: Neel Deshmukh Co-authored-by: Marcelo Villalobos Diaz --- tests/playwright-test/reporter-json.spec.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts index 79eb3c8296..27910e40ca 100644 --- a/tests/playwright-test/reporter-json.spec.ts +++ b/tests/playwright-test/reporter-json.spec.ts @@ -333,19 +333,20 @@ test('should report parallelIndex', async ({ runInlineTest }, testInfo) => { 'tests/a.spec.js': ` import { test, expect } from '@playwright/test'; const fs = require('fs'); + test.describe.configure({ mode: 'parallel' }); test('test 1 passes!', async ({}) => {}); test('test 2 fails!', async ({}) => { expect(1 + 1).toBe(3); }); test('test 3 passes!', async ({}) => {}); ` - }, { 'workers': '1', 'reporter': 'json' }); + }, { 'workers': '2', 'reporter': 'json' }); expect(result.passed).toBe(2); expect(result.failed).toBe(1); expect(result.report.suites[0].specs[0].tests[0].results[0].workerIndex).toBe(0); - expect(result.report.suites[0].specs[1].tests[0].results[0].workerIndex).toBe(0); - expect(result.report.suites[0].specs[2].tests[0].results[0].workerIndex).toBe(1); + expect(result.report.suites[0].specs[1].tests[0].results[0].workerIndex).toBe(1); + expect(result.report.suites[0].specs[2].tests[0].results[0].workerIndex).toBe(0); expect(result.report.suites[0].specs[0].tests[0].results[0].parallelIndex).toBe(0); - expect(result.report.suites[0].specs[1].tests[0].results[0].parallelIndex).toBe(0); + expect(result.report.suites[0].specs[1].tests[0].results[0].parallelIndex).toBe(1); expect(result.report.suites[0].specs[2].tests[0].results[0].parallelIndex).toBe(0); }); From 7d742ad9f4acf9e48aca40a49037ec565559e6e5 Mon Sep 17 00:00:00 2001 From: cTangonan123 Date: Wed, 26 Feb 2025 11:28:39 -0800 Subject: [PATCH 5/5] feat(json-reporter): modified test for parallelIndex to account for scheduling required when ran with 2 workers Co-authored-by: Neel Deshmukh Co-authored-by: Marcelo Villalobos Diaz --- tests/playwright-test/reporter-json.spec.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts index 27910e40ca..89b995ff35 100644 --- a/tests/playwright-test/reporter-json.spec.ts +++ b/tests/playwright-test/reporter-json.spec.ts @@ -334,19 +334,24 @@ test('should report parallelIndex', async ({ runInlineTest }, testInfo) => { import { test, expect } from '@playwright/test'; const fs = require('fs'); test.describe.configure({ mode: 'parallel' }); - test('test 1 passes!', async ({}) => {}); + test('test 1 passes!', async ({}) => { + await new Promise(f => setTimeout(f, 1000)); + }); test('test 2 fails!', async ({}) => { expect(1 + 1).toBe(3); + await new Promise(f => setTimeout(f, 1000)); + }); + test('test 3 passes!', async ({}) => { + await new Promise(f => setTimeout(f, 1000)); }); - test('test 3 passes!', async ({}) => {}); ` }, { 'workers': '2', 'reporter': 'json' }); expect(result.passed).toBe(2); expect(result.failed).toBe(1); expect(result.report.suites[0].specs[0].tests[0].results[0].workerIndex).toBe(0); expect(result.report.suites[0].specs[1].tests[0].results[0].workerIndex).toBe(1); - expect(result.report.suites[0].specs[2].tests[0].results[0].workerIndex).toBe(0); + expect(result.report.suites[0].specs[2].tests[0].results[0].workerIndex).toBe(2); expect(result.report.suites[0].specs[0].tests[0].results[0].parallelIndex).toBe(0); expect(result.report.suites[0].specs[1].tests[0].results[0].parallelIndex).toBe(1); - expect(result.report.suites[0].specs[2].tests[0].results[0].parallelIndex).toBe(0); + expect(result.report.suites[0].specs[2].tests[0].results[0].parallelIndex).toBe(1); });