Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

adjust shard.spec.ts to new algorithm

Browse source
This commit is contained in:
Mathias Leppich 2024-05-22 19:23:06 +02:00
parent ee64b15d47
commit 6543baf62f
1 changed files with 186 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

201
tests/playwright-test/shard.spec.ts
View file

@ -14,8 +14,48 @@
* limitations under the License.
*/
import { test, expect } from './playwright-test-fixtures';
import { expect, test } from './playwright-test-fixtures';
/**
* Test cases:
* - `a1.spec.ts`: 4 tests (default mode)
* - `a2.spec.ts`: 2 tests (parallel mode)
* - `a3.spec.ts`: 2 tests (parallel mode)
* - `a4.spec.ts`: 2 tests (default mode)
*
* Test Groups distribution for 2 shards
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/2: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^5
* Shard 2/2: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6 ^^^^^^^2 ^^^^^^^2
* ```
*
* Test Groups distribution for 3 shards
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/3: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
* Shard 2/3: ^^^^^^^5 ^^^^^^^2 ^^^^^^^2
* Shard 3/3: ^^^^^^^3 ^^^^^^^4 ^^^^^^^6
* ```
*
* Test Groups distribution for 4 shards
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/4: ^^^^^^^1 ^^^^^^^1 ^^^^^^^1 ^^^^^^^1
* Shard 2/4: ^^^^^^^2 ^^^^^^^2
* Shard 3/4: ^^^^^^^3 ^^^^^^^5
* Shard 4/4: ^^^^^^^4 ^^^^^^^6
* ```
*
* Test Groups distribution for 4 shards with fully-parallel
* ```
* [a1-test1, a1-test2, a1-test3, a1-test4], [a2-test1], [a2-test2], [a3-test1], [a3-test2], [a4-test1, a4-test2]
* Shard 1/4: ^^^^^^^1 ^^^^^^^5 ^^^^^^^9
* Shard 2/4: ^^^^^^^2 ^^^^^^^6 ^^^^^^10
* Shard 3/4: ^^^^^^^3 ^^^^^^^7 ^^^^^^11
* Shard 4/4: ^^^^^^^4 ^^^^^^^8
* ```
*/
const tests = {
'a1.spec.ts': `
import { test } from '@playwright/test';
@ -73,7 +113,7 @@ test('should respect shard=1/2', async ({ runInlineTest }) => {
'a1-test2-done',
'a1-test3-done',
'a1-test4-done',
'a2-test1-done',
'a3-test1-done',
]);
});
@ -83,8 +123,8 @@ test('should respect shard=2/2', async ({ runInlineTest }) => {
expect(result.passed).toBe(5);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test1-done',
'a3-test2-done',
'a4-test1-done',
'a4-test2-done',
@ -110,9 +150,9 @@ test('should respect shard=2/3', async ({ runInlineTest }) => {
expect(result.passed).toBe(3);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test1-done',
'a4-test1-done',
'a4-test2-done',
]);
});
@ -122,7 +162,31 @@ test('should respect shard=3/3', async ({ runInlineTest }) => {
expect(result.passed).toBe(3);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test2-done',
]);
});
test('should respect shard=1/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { shard: '1/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
'a1-test4-done',
]);
});
test('should respect shard=2/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { shard: '2/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a4-test1-done',
'a4-test2-done',
]);
@ -134,7 +198,18 @@ test('should respect shard=3/4', async ({ runInlineTest }) => {
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a3-test1-done',
]);
});
test('should respect shard=4/4', async ({ runInlineTest }) => {
const result = await runInlineTest(tests, { shard: '4/4', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.outputLines).toEqual([
'a2-test2-done',
'a3-test2-done',
]);
});
@ -163,7 +238,7 @@ test('should respect shard=1/2 in config', async ({ runInlineTest }) => {
'a1-test2-done',
'a1-test3-done',
'a1-test4-done',
'a2-test1-done',
'a3-test1-done',
]);
});
@ -174,20 +249,28 @@ test('should work with workers=1 and --fully-parallel', async ({ runInlineTest }
import { test } from '@playwright/test';
test('should pass', async ({ }) => {
});
`,
'a2.spec.ts': `
import { test } from '@playwright/test';
test('should pass', async ({ }) => {
});
test.skip('should skip', async ({ }) => {
});
`,
'a2.spec.ts': `
import { test } from '@playwright/test';
test('should pass', async ({ }) => {
});
`,
};
const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.skipped).toBe(1);
{
const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.skipped).toBe(1);
}
{
const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.skipped).toBe(0);
}
});
test('should skip dependency when project is sharded out', async ({ runInlineTest }) => {
@ -284,3 +367,91 @@ test('should not shard mode:default suites', async ({ runInlineTest }) => {
expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']);
}
});
test('should not shard mode:serial suites when fully-parallel', async ({ runInlineTest }) => {
const tests = {
'a1.spec.ts': `
import { test } from '@playwright/test';
test.describe.configure({ mode: 'serial' });
test('test1', async ({ }) => {
console.log('\\n%%a1-test1-done');
});
test('test2', async ({ }) => {
console.log('\\n%%a1-test2-done');
});
test('test3', async ({ }) => {
console.log('\\n%%a1-test3-done');
});
`,
'a2.spec.ts': `
import { test } from '@playwright/test';
test.describe(() => {
test('test1', async ({ }) => {
console.log('\\n%%a2-test1-done');
});
test('test2', async ({ }) => {
console.log('\\n%%a2-test2-done');
});
});
`,
'a3.spec.ts': `
import { test } from '@playwright/test';
test.describe(() => {
test('test1', async ({ }) => {
console.log('\\n%%a3-test1-done');
});
test('test2', async ({ }) => {
console.log('\\n%%a3-test2-done');
});
});
`,
};
{
const result = await runInlineTest(tests, { shard: '1/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
'a3-test2-done',
]);
}
{
const result = await runInlineTest(tests, { shard: '2/2', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual([
'a2-test1-done',
'a2-test2-done',
'a3-test1-done',
]);
}
{
const result = await runInlineTest(tests, { shard: '1/5', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.outputLines).toEqual([
'a1-test1-done',
'a1-test2-done',
'a1-test3-done',
]);
}
{
const result = await runInlineTest(tests, { shard: '2/5', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.outputLines).toEqual([
'a2-test1-done',
]);
}
{
const result = await runInlineTest(tests, { shard: '5/5', ['fully-parallel']: true, workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.outputLines).toEqual([
'a3-test2-done',
]);
}
});