diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 444a72e0df..b34f6127b0 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -267,6 +267,13 @@ An integer number that defines when the project should run relative to other pro one stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in each stage. Exeution order between projecs in the same stage is undefined. +## property: TestProject.stopOnFailure +* since: v1.28 +- type: ?<[boolean]> + +If set to true and the any test in the project fails all subsequent projects in the same playwright test run will +be skipped. + ## property: TestProject.testDir * since: v1.10 - type: ?<[string]> diff --git a/packages/playwright-test/src/dispatcher.ts b/packages/playwright-test/src/dispatcher.ts index 368833ea37..09e28c3ef5 100644 --- a/packages/playwright-test/src/dispatcher.ts +++ b/packages/playwright-test/src/dispatcher.ts @@ -30,6 +30,7 @@ export type TestGroup = { requireFile: string; repeatEachIndex: number; projectId: string; + stopOnFailure: boolean; tests: TestCase[]; watchMode: boolean; }; diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 5459cc4ab1..44d6767900 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -278,6 +278,7 @@ export class Loader { const snapshotDir = takeFirst(projectConfig.snapshotDir, config.snapshotDir, testDir); const name = takeFirst(projectConfig.name, config.name, ''); const stage = takeFirst(projectConfig.stage, 0); + const stopOnFailure = takeFirst(projectConfig.stopOnFailure, false); let screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name)); if (process.env.PLAYWRIGHT_DOCKER) { @@ -298,6 +299,7 @@ export class Loader { name, testDir, stage, + stopOnFailure, _respectGitIgnore: respectGitIgnore, snapshotDir, _screenshotsDir: screenshotsDir, @@ -609,6 +611,16 @@ function validateProject(file: string, project: Project, title: string) { throw errorWithFile(file, `${title}.retries must be a non-negative number`); } + if ('stage' in project && project.stage !== undefined) { + if (typeof project.stage !== 'number' || Math.floor(project.stage) !== project.stage) + throw errorWithFile(file, `${title}.stage must be an integer`); + } + + if ('stopOnFailure' in project && project.stopOnFailure !== undefined) { + if (typeof project.stopOnFailure !== 'boolean') + throw errorWithFile(file, `${title}.stopOnFailure must be a boolean`); + } + if ('testDir' in project && project.testDir !== undefined) { if (typeof project.testDir !== 'string') throw errorWithFile(file, `${title}.testDir must be a string`); diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index ea4a6e7767..878d45b0d2 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -426,7 +426,7 @@ export class Runner { let hasWorkerErrors = false; for (const testGroups of concurrentTestGroups) { - const dispatcher = new Dispatcher(this._loader, testGroups, this._reporter); + const dispatcher = new Dispatcher(this._loader, [...testGroups], this._reporter); sigintWatcher = new SigIntWatcher(); await Promise.race([dispatcher.run(), sigintWatcher.promise()]); if (!sigintWatcher.hadSignal()) { @@ -438,7 +438,8 @@ export class Runner { hasWorkerErrors = dispatcher.hasWorkerErrors(); if (hasWorkerErrors) break; - if (testGroups.some(testGroup => testGroup.tests.some(test => !test.ok()))) + const stopOnFailureGroups = testGroups.filter(group => group.stopOnFailure); + if (stopOnFailureGroups.some(testGroup => testGroup.tests.some(test => !test.ok()))) break; if (sigintWatcher.hadSignal()) break; @@ -747,6 +748,7 @@ function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[] requireFile: test._requireFile, repeatEachIndex: test.repeatEachIndex, projectId: test._projectId, + stopOnFailure: test.parent.project()!.stopOnFailure, tests: [], watchMode: false, }; diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index bb1d6c72b0..54cfeeca69 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -262,6 +262,11 @@ export interface FullProject { * stage. Exeution order between projecs in the same stage is undefined. */ stage: number; + /** + * If set to true and the any test in the project fails all subsequent projects in the same playwright test run will be + * skipped. + */ + stopOnFailure: boolean; /** * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file. * @@ -4471,6 +4476,12 @@ interface TestProject { */ stage?: number; + /** + * If set to true and the any test in the project fails all subsequent projects in the same playwright test run will be + * skipped. + */ + stopOnFailure?: boolean; + /** * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file. * diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index 23082e0a60..541d58f133 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -480,3 +480,60 @@ test('should have correct types for the config', async ({ runTSC }) => { }); expect(result.exitCode).toBe(0); }); + +test('should throw when project.stage is not a number', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'a', stage: 'foo' }, + ], + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async () => {}); + ` + }); + + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`config.projects[0].stage must be an integer`); +}); + +test('should throw when project.stage is not an integer', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'a', stage: 3.14 }, + ], + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async () => {}); + ` + }); + + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`config.projects[0].stage must be an integer`); +}); + +test('should throw when project.stopOnFailure is not a boolean', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'a', stopOnFailure: 'yes' }, + ], + }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async () => {}); + ` + }); + + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`config.projects[0].stopOnFailure must be a boolean`); +}); diff --git a/tests/playwright-test/stages.spec.ts b/tests/playwright-test/stages.spec.ts index 569d529a67..4632750753 100644 --- a/tests/playwright-test/stages.spec.ts +++ b/tests/playwright-test/stages.spec.ts @@ -206,4 +206,82 @@ test('should work with project filter', async ({ runGroups }, testInfo) => { expectRunBefore(timeline, ['e'], ['b', 'c']); // -10 < 0 expectRunBefore(timeline, ['c'], ['b']); // 0 < 10 expect(passed).toBe(3); -}); \ No newline at end of file +}); + +test('should continue after failures', async ({ runGroups }, testInfo) => { + const projectTemplates = { + 'a': { + stage: 1 + }, + 'b': { + stage: 2 + }, + 'c': { + stage: 2 + }, + 'd': { + stage: 4 + }, + 'e': { + stage: 4 + }, + }; + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e'], testInfo, projectTemplates); + configWithFiles[`b/b.spec.ts`] = ` + const { test } = pwt; + test('b test', async () => { + expect(1).toBe(2); + });`; + configWithFiles[`d/d.spec.ts`] = ` + const { test } = pwt; + test('d test', async () => { + expect(1).toBe(2); + });`; + const { exitCode, passed, failed, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(1); + expect(failed).toBe(2); + expect(passed).toBe(3); + expect(projectNames(timeline)).toEqual(['a', 'b', 'c', 'd', 'e']); + expectRunBefore(timeline, ['a'], ['b', 'c', 'd', 'e']); // 1 < 2 + expectRunBefore(timeline, ['b', 'c'], ['d', 'e']); // 2 < 4 +}); + +test('should support stopOnFailire', async ({ runGroups }, testInfo) => { + const projectTemplates = { + 'a': { + stage: 1 + }, + 'b': { + stage: 2, + stopOnFailure: true + }, + 'c': { + stage: 2 + }, + 'd': { + stage: 4, + stopOnFailure: true // this is not important as the test is skipped + }, + 'e': { + stage: 4 + }, + }; + const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e'], testInfo, projectTemplates); + configWithFiles[`b/b.spec.ts`] = ` + const { test } = pwt; + test('b test', async () => { + expect(1).toBe(2); + });`; + configWithFiles[`d/d.spec.ts`] = ` + const { test } = pwt; + test('d test', async () => { + expect(1).toBe(2); + });`; + const { exitCode, passed, failed, skipped, timeline } = await runGroups(configWithFiles); + expect(exitCode).toBe(1); + expect(failed).toBe(1); + expect(passed).toBeLessThanOrEqual(2); // 'c' may either pass or be skipped. + expect(passed + skipped).toBe(4); + expect(projectNames(timeline)).not.toContainEqual(['d', 'e']); +}); + diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index c6a4886a73..c7957636fb 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -47,6 +47,7 @@ export interface FullProject { repeatEach: number; retries: number; stage: number; + stopOnFailure: boolean; testDir: string; testIgnore: string | RegExp | (string | RegExp)[]; testMatch: string | RegExp | (string | RegExp)[];