diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.ts index e82a434f52..9c3839aaa8 100644 --- a/tests/playwright-test/shard.spec.ts +++ b/tests/playwright-test/shard.spec.ts @@ -14,8 +14,48 @@ * limitations under the License. */ -import { test, expect } from './playwright-test-fixtures'; +import { expect, test } from './playwright-test-fixtures'; +/** + * Test cases: + * - `a1.spec.ts`: 4 tests (default mode) + * - `a2.spec.ts`: 2 tests (parallel mode) + * - `a3.spec.ts`: 2 tests (parallel mode) + * - `a4.spec.ts`: 2 tests (default mode) + * + * Test Groups distribution for 2 shards + * ``` + * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] + * Shard 1/2: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^5 + * Shard 2/2: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 ^^^^^^^2 ^^^^^^^2 + * ``` + * + * Test Groups distribution for 3 shards + * ``` + * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] + * Shard 1/3: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 + * Shard 2/3: ^^^^^^^5 ^^^^^^^2 ^^^^^^^2 + * Shard 3/3: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 + * ``` + * + * Test Groups distribution for 4 shards + * ``` + * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] + * Shard 1/4: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 + * Shard 2/4: ^^^^^^^2 ^^^^^^^2 + * Shard 3/4: ^^^^^^^3 ^^^^^^^5 + * Shard 4/4: ^^^^^^^4 ^^^^^^^6 + * ``` + * + * Test Groups distribution for 4 shards with fully-parallel + * ``` + * [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2] + * Shard 1/4: ^^^^^^^1 ^^^^^^^5 ^^^^^^^9 + * Shard 2/4: ^^^^^^^2 ^^^^^^^6 ^^^^^^10 + * Shard 3/4: ^^^^^^^3 ^^^^^^^7 ^^^^^^11 + * Shard 4/4: ^^^^^^^4 ^^^^^^^8 + * ``` + */ const tests = { 'a1.spec.ts': ` import { test } from '@playwright/test'; @@ -73,7 +113,7 @@ test('should respect shard=1/2', async ({ runInlineTest }) => { 'a1-test2-done', 'a1-test3-done', 'a1-test4-done', - 'a2-test1-done', + 'a3-test1-done', ]); }); @@ -83,8 +123,8 @@ test('should respect shard=2/2', async ({ runInlineTest }) => { expect(result.passed).toBe(5); expect(result.skipped).toBe(0); expect(result.outputLines).toEqual([ + 'a2-test1-done', 'a2-test2-done', - 'a3-test1-done', 'a3-test2-done', 'a4-test1-done', 'a4-test2-done', @@ -110,9 +150,9 @@ test('should respect shard=2/3', async ({ runInlineTest }) => { expect(result.passed).toBe(3); expect(result.skipped).toBe(0); expect(result.outputLines).toEqual([ - 'a2-test1-done', - 'a2-test2-done', 'a3-test1-done', + 'a4-test1-done', + 'a4-test2-done', ]); }); @@ -122,7 +162,31 @@ test('should respect shard=3/3', async ({ runInlineTest }) => { expect(result.passed).toBe(3); expect(result.skipped).toBe(0); expect(result.outputLines).toEqual([ + 'a2-test1-done', + 'a2-test2-done', 'a3-test2-done', + ]); +}); + +test('should respect shard=1/4', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { shard: '1/4', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(4); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a1-test1-done', + 'a1-test2-done', + 'a1-test3-done', + 'a1-test4-done', + ]); +}); + +test('should respect shard=2/4', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { shard: '2/4', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ 'a4-test1-done', 'a4-test2-done', ]); @@ -134,7 +198,18 @@ test('should respect shard=3/4', async ({ runInlineTest }) => { expect(result.passed).toBe(2); expect(result.skipped).toBe(0); expect(result.outputLines).toEqual([ + 'a2-test1-done', 'a3-test1-done', + ]); +}); + +test('should respect shard=4/4', async ({ runInlineTest }) => { + const result = await runInlineTest(tests, { shard: '4/4', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.skipped).toBe(0); + expect(result.outputLines).toEqual([ + 'a2-test2-done', 'a3-test2-done', ]); }); @@ -163,7 +238,7 @@ test('should respect shard=1/2 in config', async ({ runInlineTest }) => { 'a1-test2-done', 'a1-test3-done', 'a1-test4-done', - 'a2-test1-done', + 'a3-test1-done', ]); }); @@ -174,20 +249,28 @@ test('should work with workers=1 and --fully-parallel', async ({ runInlineTest } import { test } from '@playwright/test'; test('should pass', async ({ }) => { }); + `, + 'a2.spec.ts': ` + import { test } from '@playwright/test'; + test('should pass', async ({ }) => { + }); test.skip('should skip', async ({ }) => { }); `, - 'a2.spec.ts': ` - import { test } from '@playwright/test'; - test('should pass', async ({ }) => { - }); - `, }; - const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.skipped).toBe(1); + { + const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.skipped).toBe(1); + } + { + const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.skipped).toBe(0); + } }); test('should skip dependency when project is sharded out', async ({ runInlineTest }) => { @@ -284,3 +367,91 @@ test('should not shard mode:default suites', async ({ runInlineTest }) => { expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']); } }); + +test('should not shard mode:serial suites when fully-parallel', async ({ runInlineTest }) => { + const tests = { + 'a1.spec.ts': ` + import { test } from '@playwright/test'; + test.describe.configure({ mode: 'serial' }); + test('test1', async ({ }) => { + console.log('\\n%%a1-test1-done'); + }); + test('test2', async ({ }) => { + console.log('\\n%%a1-test2-done'); + }); + test('test3', async ({ }) => { + console.log('\\n%%a1-test3-done'); + }); + `, + 'a2.spec.ts': ` + import { test } from '@playwright/test'; + test.describe(() => { + test('test1', async ({ }) => { + console.log('\\n%%a2-test1-done'); + }); + test('test2', async ({ }) => { + console.log('\\n%%a2-test2-done'); + }); + }); + `, + 'a3.spec.ts': ` + import { test } from '@playwright/test'; + test.describe(() => { + test('test1', async ({ }) => { + console.log('\\n%%a3-test1-done'); + }); + test('test2', async ({ }) => { + console.log('\\n%%a3-test2-done'); + }); + }); + `, + }; + + { + const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(4); + expect(result.outputLines).toEqual([ + 'a1-test1-done', + 'a1-test2-done', + 'a1-test3-done', + 'a3-test2-done', + ]); + } + { + const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.outputLines).toEqual([ + 'a2-test1-done', + 'a2-test2-done', + 'a3-test1-done', + ]); + } + { + const result = await runInlineTest(tests, { shard: '1/5', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(result.outputLines).toEqual([ + 'a1-test1-done', + 'a1-test2-done', + 'a1-test3-done', + ]); + } + { + const result = await runInlineTest(tests, { shard: '2/5', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.outputLines).toEqual([ + 'a2-test1-done', + ]); + } + { + const result = await runInlineTest(tests, { shard: '5/5', ['fully-parallel']: true, workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.outputLines).toEqual([ + 'a3-test2-done', + ]); + } +});