Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(runner): project.setup (#18486)

Browse source
This commit is contained in:
Yury Semikhatsky 2022-11-01 23:44:30 -07:00 committed by GitHub
parent 67c9624924
commit 1d2fc1e963
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 492 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
docs/src/test-api/class-testproject.md
View file

@ -162,6 +162,11 @@ Metadata that will be put directly to the test report serialized as JSON.
Project name is visible in the report and during test execution.
## property: TestProject.setup
* since: v1.28
- type: ?<[string]|[RegExp]|[Array]<[string]|[RegExp]>>
Project setup files that would be executed before all tests in the project. If project setup fails the tests in this project will be skipped. All project setup files will run in every shard if the project is sharded.
## property: TestProject.screenshotsDir
* since: v1.10

1
packages/playwright-test/src/dispatcher.ts
View file

@ -30,7 +30,6 @@ export type TestGroup = {
requireFile: string;
repeatEachIndex: number;
projectId: string;
run: 'default'|'always';
tests: TestCase[];
watchMode: boolean;
};

4
packages/playwright-test/src/loader.ts
View file

@ -277,6 +277,7 @@ export class Loader {
const outputDir = takeFirst(projectConfig.outputDir, config.outputDir, path.join(throwawayArtifactsPath, 'test-results'));
const snapshotDir = takeFirst(projectConfig.snapshotDir, config.snapshotDir, testDir);
const name = takeFirst(projectConfig.name, config.name, '');
const _setup = takeFirst(projectConfig.setup, []);
let screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name));
if (process.env.PLAYWRIGHT_DOCKER) {
@ -296,6 +297,7 @@ export class Loader {
metadata: takeFirst(projectConfig.metadata, config.metadata, undefined),
name,
testDir,
_setup,
_respectGitIgnore: respectGitIgnore,
snapshotDir,
_screenshotsDir: screenshotsDir,
@ -618,7 +620,7 @@ function validateProject(file: string, project: Project, title: string) {
throw errorWithFile(file, `${title}.testDir must be a string`);
}
for (const prop of ['testIgnore', 'testMatch'] as const) {
for (const prop of ['testIgnore', 'testMatch', 'setup'] as const) {
if (prop in project && project[prop] !== undefined) {
const value = project[prop];
if (Array.isArray(value)) {

179
packages/playwright-test/src/runner.ts
View file

@ -196,7 +196,7 @@ export class Runner {
async listTestFiles(configFile: string, projectNames: string[] | undefined): Promise<any> {
const projects = this._collectProjects(projectNames);
const filesByProject = await this._collectFiles(projects, () => true);
const { filesByProject } = await this._collectFiles(projects, () => true);
const report: any = {
projects: []
};
@ -237,24 +237,48 @@ export class Runner {
return projects;
}
private async _collectFiles(projects: FullProjectInternal[], testFileFilter: Matcher): Promise<Map<FullProjectInternal, string[]>> {
const files = new Map<FullProjectInternal, string[]>();
private async _collectFiles(projects: FullProjectInternal[], testFileFilter: Matcher): Promise<{filesByProject: Map<FullProjectInternal, string[]>; setupFiles: Set<string>}> {
const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
const testFileExtension = (file: string) => extensions.includes(path.extname(file));
const filesByProject = new Map<FullProjectInternal, string[]>();
const setupFiles = new Set<string>();
const fileToProjectName = new Map<string, string>();
for (const project of projects) {
const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
const setupMatch = createFileMatcher(project._setup);
const testMatch = createFileMatcher(project.testMatch);
const testIgnore = createFileMatcher(project.testIgnore);
const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
const testFileExtension = (file: string) => extensions.includes(path.extname(file));
const testFiles = allFiles.filter(file => !testIgnore(file) && testMatch(file) && testFileFilter(file) && testFileExtension(file));
files.set(project, testFiles);
const testFiles = allFiles.filter(file => {
if (!testFileExtension(file))
return false;
const isSetup = setupMatch(file);
const isTest = !testIgnore(file) && testMatch(file) && testFileFilter(file);
if (!isTest && !isSetup)
return false;
if (isSetup && isTest)
throw new Error(`File "${file}" matches both 'setup' and 'testMatch' filters in project "${project.name}"`);
if (fileToProjectName.has(file)) {
if (isSetup) {
if (!setupFiles.has(file))
throw new Error(`File "${file}" matches 'setup' filter in project "${project.name}" and 'testMatch' filter in project "${fileToProjectName.get(file)}"`);
} else if (setupFiles.has(file)) {
throw new Error(`File "${file}" matches 'setup' filter in project "${fileToProjectName.get(file)}" and 'testMatch' filter in project "${project.name}"`);
}
}
fileToProjectName.set(file, project.name);
if (isSetup)
setupFiles.add(file);
return true;
});
filesByProject.set(project, testFiles);
}
return files;
return { filesByProject, setupFiles };
}
private async _collectTestGroups(options: RunOptions, fatalErrors: TestError[]): Promise<{ rootSuite: Suite, testGroups: TestGroup[] }> {
private async _collectTestGroups(options: RunOptions, fatalErrors: TestError[]): Promise<{ rootSuite: Suite, projectSetupGroups: TestGroup[], testGroups: TestGroup[] }> {
const config = this._loader.fullConfig();
const projects = this._collectProjects(options.projectFilter);
const filesByProject = await this._collectFiles(projects, fileMatcherFrom(options.testFileFilters));
const { filesByProject, setupFiles } = await this._collectFiles(projects, fileMatcherFrom(options.testFileFilters));
const allTestFiles = new Set<string>();
for (const files of filesByProject.values())
@ -276,9 +300,10 @@ export class Runner {
// Filter tests to respect line/column filter.
if (options.testFileFilters.length)
filterByFocusedLine(preprocessRoot, options.testFileFilters);
filterByFocusedLine(preprocessRoot, options.testFileFilters, setupFiles);
// Complain about only.
// TODO: check in project setup.
if (config.forbidOnly) {
const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
if (onlyTestsAndSuites.length > 0)
@ -321,23 +346,31 @@ export class Runner {
}
}
const testGroups = createTestGroups(rootSuite.suites, config.workers);
return { rootSuite, testGroups };
const allTestGroups = createTestGroups(rootSuite.suites, config.workers);
const projectSetupGroups = [];
const testGroups = [];
for (const group of allTestGroups) {
if (setupFiles.has(group.requireFile))
projectSetupGroups.push(group);
else
testGroups.push(group);
}
return { rootSuite, projectSetupGroups, testGroups };
}
private _filterForCurrentShard(rootSuite: Suite, testGroups: TestGroup[]) {
private _filterForCurrentShard(rootSuite: Suite, projectSetupGroups: TestGroup[], testGroups: TestGroup[]) {
const shard = this._loader.fullConfig().shard;
if (!shard)
return;
// Each shard includes:
// - all tests from `run: 'always'` projects (non shardale) and
// - its portion of the shardable ones.
// - its portion of the regular tests
// - project setup tests for the projects that have regular tests in this shard
let shardableTotal = 0;
for (const group of testGroups) {
if (group.run !== 'always')
shardableTotal += group.tests.length;
}
for (const group of testGroups)
shardableTotal += group.tests.length;
const shardTests = new Set<TestCase>();
@ -350,27 +383,33 @@ export class Runner {
const from = shardSize * currentShard + Math.min(extraOne, currentShard);
const to = from + shardSize + (currentShard < extraOne ? 1 : 0);
let current = 0;
const shardProjects = new Set<string>();
const shardTestGroups = [];
for (const group of testGroups) {
let includeGroupInShard = false;
if (group.run === 'always') {
includeGroupInShard = true;
} else {
// Any test group goes to the shard that contains the first test of this group.
// So, this shard gets any group that starts at [from; to)
if (current >= from && current < to)
includeGroupInShard = true;
current += group.tests.length;
}
if (includeGroupInShard) {
// Any test group goes to the shard that contains the first test of this group.
// So, this shard gets any group that starts at [from; to)
if (current >= from && current < to) {
shardProjects.add(group.projectId);
shardTestGroups.push(group);
for (const test of group.tests)
shardTests.add(test);
}
current += group.tests.length;
}
testGroups.length = 0;
testGroups.push(...shardTestGroups);
const shardSetupGroups = [];
for (const group of projectSetupGroups) {
if (!shardProjects.has(group.projectId))
continue;
shardSetupGroups.push(group);
for (const test of group.tests)
shardTests.add(test);
}
projectSetupGroups.length = 0;
projectSetupGroups.push(...shardSetupGroups);
filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test));
}
@ -379,15 +418,15 @@ export class Runner {
const fatalErrors: TestError[] = [];
// Each entry is an array of test groups that can be run concurrently. All
// test groups from the previos entries must finish before entry starts.
const { rootSuite, testGroups } = await this._collectTestGroups(options, fatalErrors);
const { rootSuite, projectSetupGroups, testGroups } = await this._collectTestGroups(options, fatalErrors);
// Fail when no tests.
if (!rootSuite.allTests().length && !options.passWithNoTests)
fatalErrors.push(createNoTestsError());
this._filterForCurrentShard(rootSuite, testGroups);
this._filterForCurrentShard(rootSuite, projectSetupGroups, testGroups);
config._maxConcurrentTestGroups = testGroups.length;
config._maxConcurrentTestGroups = Math.max(projectSetupGroups.length, testGroups.length);
// Report begin
this._reporter.onBegin?.(config, rootSuite);
@ -424,37 +463,20 @@ export class Runner {
// Run tests.
try {
let sigintWatcher;
let hasWorkerErrors = false;
let previousStageFailed = false;
// TODO: will be project setups followed by tests.
const orderedTestGroups = [testGroups];
for (let testGroups of orderedTestGroups) {
if (previousStageFailed)
testGroups = this._skipTestsNotMarkedAsRunAlways(testGroups);
if (!testGroups.length)
continue;
const dispatcher = new Dispatcher(this._loader, [...testGroups], this._reporter);
sigintWatcher = new SigIntWatcher();
await Promise.race([dispatcher.run(), sigintWatcher.promise()]);
if (!sigintWatcher.hadSignal()) {
// We know for sure there was no Ctrl+C, so we remove custom SIGINT handler
// as soon as we can.
sigintWatcher.disarm();
let dispatchResult = await this._dispatchToWorkers(projectSetupGroups);
if (dispatchResult === 'success') {
const failedSetupProjectIds = new Set<string>();
for (const testGroup of projectSetupGroups) {
if (testGroup.tests.some(test => !test.ok()))
failedSetupProjectIds.add(testGroup.projectId);
}
await dispatcher.stop();
hasWorkerErrors = dispatcher.hasWorkerErrors();
if (hasWorkerErrors)
break;
if (sigintWatcher.hadSignal())
break;
previousStageFailed ||= testGroups.some(testGroup => testGroup.tests.some(test => !test.ok()));
const testGroupsToRun = this._skipTestsFromFailedProjects(testGroups, failedSetupProjectIds);
dispatchResult = await this._dispatchToWorkers(testGroupsToRun);
}
if (sigintWatcher?.hadSignal()) {
if (dispatchResult === 'signal') {
result.status = 'interrupted';
} else {
const failed = hasWorkerErrors || rootSuite.allTests().some(test => !test.ok());
const failed = dispatchResult === 'workererror' || rootSuite.allTests().some(test => !test.ok());
result.status = failed ? 'failed' : 'passed';
}
} catch (e) {
@ -466,21 +488,38 @@ export class Runner {
return result;
}
private _skipTestsNotMarkedAsRunAlways(testGroups: TestGroup[]): TestGroup[] {
const runAlwaysGroups = [];
private async _dispatchToWorkers(stageGroups: TestGroup[]): Promise<'success'|'signal'|'workererror'> {
const dispatcher = new Dispatcher(this._loader, [...stageGroups], this._reporter);
const sigintWatcher = new SigIntWatcher();
await Promise.race([dispatcher.run(), sigintWatcher.promise()]);
if (!sigintWatcher.hadSignal()) {
// We know for sure there was no Ctrl+C, so we remove custom SIGINT handler
// as soon as we can.
sigintWatcher.disarm();
}
await dispatcher.stop();
if (sigintWatcher.hadSignal())
return 'signal';
if (dispatcher.hasWorkerErrors())
return 'workererror';
return 'success';
}
private _skipTestsFromFailedProjects(testGroups: TestGroup[], failedProjects: Set<string>): TestGroup[] {
const result = [];
for (const group of testGroups) {
if (group.run === 'always') {
runAlwaysGroups.push(group);
} else {
if (failedProjects.has(group.projectId)) {
for (const test of group.tests) {
const result = test._appendTestResult();
this._reporter.onTestBegin?.(test, result);
result.status = 'skipped';
this._reporter.onTestEnd?.(test, result);
}
} else {
result.push(group);
}
}
return runAlwaysGroups;
return result;
}
private async _removeOutputDirs(options: RunOptions): Promise<boolean> {
@ -595,7 +634,7 @@ function filterOnly(suite: Suite) {
return filterSuiteWithOnlySemantics(suite, suiteFilter, testFilter);
}
function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[]) {
function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[], setupFiles: Set<string>) {
const filterWithLine = !!focusedTestFileLines.find(f => f.line !== null);
if (!filterWithLine)
return;
@ -609,7 +648,8 @@ function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[
const suiteFilter = (suite: Suite) => {
return !!suite.location && testFileLineMatches(suite.location.file, suite.location.line, suite.location.column);
};
const testFilter = (test: TestCase) => testFileLineMatches(test.location.file, test.location.line, test.location.column);
// Project setup files are always included.
const testFilter = (test: TestCase) => setupFiles.has(test._requireFile) || testFileLineMatches(test.location.file, test.location.line, test.location.column);
return filterSuite(suite, suiteFilter, testFilter);
}
@ -754,7 +794,6 @@ function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[]
requireFile: test._requireFile,
repeatEachIndex: test.repeatEachIndex,
projectId: test._projectId,
run: 'default',
tests: [],
watchMode: false,
};

1
packages/playwright-test/src/types.ts
View file

@ -68,6 +68,7 @@ export interface FullProjectInternal extends FullProjectPublic {
_expect: Project['expect'];
_screenshotsDir: string;
_respectGitIgnore: boolean;
_setup: string | RegExp | (string | RegExp)[];
}
export interface ReporterInternal extends Reporter {

6
packages/playwright-test/types/test.d.ts vendored
View file

@ -4414,6 +4414,12 @@ interface TestProject {
*/
name?: string;
/**
* Project setup files that would be executed before all tests in the project. If project setup fails the tests in this
* project will be skipped. All project setup files will run in every shard if the project is sharded.
*/
setup?: string|RegExp|Array<string|RegExp>;
/**
* The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to
* [testProject.testDir](https://playwright.dev/docs/api/class-testproject#test-project-test-dir).

38
tests/playwright-test/config.spec.ts
View file

@ -480,3 +480,41 @@ test('should have correct types for the config', async ({ runTSC }) => {
});
expect(result.exitCode).toBe(0);
});
test('should throw when project.setup has wrong type', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'a', setup: 100 },
],
};
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async () => {});
`
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain(`Error: playwright.config.ts: config.projects[0].setup must be a string or a RegExp`);
});
test('should throw when project.setup has wrong array type', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'a', setup: [/100/, 100] },
],
};
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async () => {});
`
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain(`Error: playwright.config.ts: config.projects[0].setup[1] must be a string or a RegExp`);
});

330
tests/playwright-test/project-setup.spec.ts Normal file
View file

@ -0,0 +1,330 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { PlaywrightTestConfig, TestInfo, PlaywrightTestProject } from '@playwright/test';
import path from 'path';
import { test, expect } from './playwright-test-fixtures';
function createConfigWithProjects(names: string[], testInfo: TestInfo, projectTemplates?: { [name: string]: PlaywrightTestProject }): Record<string, string> {
const config: PlaywrightTestConfig = {
projects: names.map(name => ({ ...projectTemplates?.[name], name, testDir: testInfo.outputPath(name) })),
};
const files = {};
for (const name of names) {
files[`${name}/${name}.spec.ts`] = `
const { test } = pwt;
test('${name} test', async () => {
await new Promise(f => setTimeout(f, 100));
});`;
files[`${name}/${name}.setup.ts`] = `
const { test } = pwt;
test('${name} setup', async () => {
await new Promise(f => setTimeout(f, 100));
});`;
}
function replacer(key, value) {
if (value instanceof RegExp)
return `RegExp(${value.toString()})`;
else
return value;
}
files['playwright.config.ts'] = `
import * as path from 'path';
module.exports = ${JSON.stringify(config, replacer, 2)};
`.replace(/"RegExp\((.*)\)"/g, '$1');
return files;
}
type Timeline = { titlePath: string[], event: 'begin' | 'end' }[];
function formatTimeline(timeline: Timeline) {
return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n');
}
function formatFileNames(timeline: Timeline) {
return timeline.map(e => e.titlePath[2]).join('\n');
}
function fileNames(timeline: Timeline) {
const fileNames = Array.from(new Set(timeline.map(({ titlePath }) => {
const name = titlePath[2];
const index = name.lastIndexOf(path.sep);
if (index === -1)
return name;
return name.slice(index + 1);
})).keys());
fileNames.sort();
return fileNames;
}
function expectFilesRunBefore(timeline: Timeline, before: string[], after: string[]) {
const fileBegin = name => {
const index = timeline.findIndex(({ titlePath }) => titlePath[2] === name);
expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1);
return index;
};
const fileEnd = name => {
// There is no Array.findLastIndex in Node < 18.
let index = -1;
for (index = timeline.length - 1; index >= 0; index--) {
if (timeline[index].titlePath[2] === name)
break;
}
expect(index, `cannot find ${name} in\n${formatFileNames(timeline)}`).not.toBe(-1);
return index;
};
for (const b of before) {
const bEnd = fileEnd(b);
for (const a of after) {
const aBegin = fileBegin(a);
expect(bEnd < aBegin, `'${b}' expected to finish before ${a}, actual order:\n${formatTimeline(timeline)}`).toBeTruthy();
}
}
}
test('should work for one project', async ({ runGroups }, testInfo) => {
const projectTemplates = {
'a': {
setup: ['**/*.setup.ts']
},
};
const configWithFiles = createConfigWithProjects(['a'], testInfo, projectTemplates);
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
expect(passed).toBe(2);
expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.setup.ts > a setup [begin]
a > a${path.sep}a.setup.ts > a setup [end]
a > a${path.sep}a.spec.ts > a test [begin]
a > a${path.sep}a.spec.ts > a test [end]`);
});
test('should work for several projects', async ({ runGroups }, testInfo) => {
const projectTemplates = {
'a': {
setup: ['**/*.setup.ts']
},
'b': {
setup: /.*b.setup.ts/
},
'c': {
setup: '**/c.setup.ts'
},
};
const configWithFiles = createConfigWithProjects(['a', 'b', 'c'], testInfo, projectTemplates);
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
expect(exitCode).toBe(0);
expect(passed).toBe(6);
for (const name of ['a', 'b', 'c'])
expectFilesRunBefore(timeline, [`${name}${path.sep}${name}.setup.ts`], [`${name}${path.sep}${name}.spec.ts`]);
});
test('should stop project if setup fails', async ({ runGroups }, testInfo) => {
const projectTemplates = {
'a': {
setup: ['**/*.setup.ts']
},
'b': {
setup: /.*b.setup.ts/
},
};
const configWithFiles = createConfigWithProjects(['a', 'b', 'c'], testInfo, projectTemplates);
configWithFiles[`a/a.setup.ts`] = `
const { test, expect } = pwt;
test('a setup', async () => {
expect(1).toBe(2);
});`;
const { exitCode, passed, skipped, timeline } = await runGroups(configWithFiles);
expect(exitCode).toBe(1);
expect(passed).toBe(3);
expect(skipped).toBe(1); // 1 test from project 'a'
for (const name of ['a', 'b'])
expectFilesRunBefore(timeline, [`${name}${path.sep}${name}.setup.ts`], [`${name}${path.sep}${name}.spec.ts`]);
});
test('should run setup in each project shard', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*.setup.ts/,
},
]
};`,
'a.test.ts': `
const { test } = pwt;
test('test1', async () => { });
test('test2', async () => { });
test('test3', async () => { });
test('test4', async () => { });
`,
'b.test.ts': `
const { test } = pwt;
test('test1', async () => { });
test('test2', async () => { });
`,
'c.setup.ts': `
const { test } = pwt;
test('setup1', async () => { });
test('setup2', async () => { });
`,
};
{ // Shard 1/2
const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '1/2' });
expect(output).toContain('Running 6 tests using 1 worker, shard 1 of 2');
expect(fileNames(timeline)).toEqual(['a.test.ts', 'c.setup.ts']);
expectFilesRunBefore(timeline, [`c.setup.ts`], [`a.test.ts`]);
expect(exitCode).toBe(0);
expect(passed).toBe(6);
}
{ // Shard 2/2
const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '2/2' });
expect(output).toContain('Running 4 tests using 1 worker, shard 2 of 2');
expect(fileNames(timeline)).toEqual(['b.test.ts', 'c.setup.ts']);
expectFilesRunBefore(timeline, [`c.setup.ts`], [`b.test.ts`]);
expect(exitCode).toBe(0);
expect(passed).toBe(4);
}
});
test('should run setup only for projects that have tests in the shard', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*p1.setup.ts$/,
testMatch: /.*a.test.ts/,
},
{
name: 'p2',
setup: /.*p2.setup.ts$/,
testMatch: /.*b.test.ts/,
},
]
};`,
'a.test.ts': `
const { test } = pwt;
test('test1', async () => { });
test('test2', async () => { });
test('test3', async () => { });
test('test4', async () => { });
`,
'b.test.ts': `
const { test } = pwt;
test('test1', async () => { });
test('test2', async () => { });
`,
'p1.setup.ts': `
const { test } = pwt;
test('setup1', async () => { });
test('setup2', async () => { });
`,
'p2.setup.ts': `
const { test } = pwt;
test('setup3', async () => { });
test('setup4', async () => { });
`,
};
{ // Shard 1/2
const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '1/2' });
expect(output).toContain('Running 6 tests using 1 worker, shard 1 of 2');
expect(fileNames(timeline)).toEqual(['a.test.ts', 'p1.setup.ts']);
expectFilesRunBefore(timeline, [`p1.setup.ts`], [`a.test.ts`]);
expect(exitCode).toBe(0);
expect(passed).toBe(6);
}
{ // Shard 2/2
const { exitCode, passed, timeline, output } = await runGroups(files, { shard: '2/2' });
expect(output).toContain('Running 4 tests using 1 worker, shard 2 of 2');
expect(fileNames(timeline)).toEqual(['b.test.ts', 'p2.setup.ts']);
expectFilesRunBefore(timeline, [`p2.setup.ts`], [`b.test.ts`]);
expect(exitCode).toBe(0);
expect(passed).toBe(4);
}
});
test('--project only runs setup from that project;', async ({ runGroups }, testInfo) => {
const projectTemplates = {
'a': {
setup: /.*a.setup.ts/
},
'b': {
setup: /.*b.setup.ts/
},
};
const configWithFiles = createConfigWithProjects(['a', 'b', 'c'], testInfo, projectTemplates);
const { exitCode, passed, timeline } = await runGroups(configWithFiles, { project: ['a', 'c'] });
expect(exitCode).toBe(0);
expect(passed).toBe(3);
expect(fileNames(timeline)).toEqual(['a.setup.ts', 'a.spec.ts', 'c.spec.ts']);
});
test('same file cannot be a setup and a test in the same project', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*a.test.ts$/,
testMatch: /.*a.test.ts$/,
},
]
};`,
'a.test.ts': `
const { test } = pwt;
test('test1', async () => { });
`,
};
const { exitCode, output } = await runGroups(files);
expect(exitCode).toBe(1);
expect(output).toContain(`a.test.ts" matches both 'setup' and 'testMatch' filters in project "p1"`);
});
test('same file cannot be a setup and a test in different projects', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
projects: [
{
name: 'p1',
setup: /.*a.test.ts$/,
testMatch: /.*noMatch.test.ts$/,
},
{
name: 'p2',
setup: /.*noMatch.test.ts$/,
testMatch: /.*a.test.ts$/
},
]
};`,
'a.test.ts': `
const { test } = pwt;
test('test1', async () => { });
`,
};
const { exitCode, output } = await runGroups(files);
expect(exitCode).toBe(1);
expect(output).toContain(`a.test.ts" matches 'setup' filter in project "p1" and 'testMatch' filter in project "p2"`);
});