feat(runner): project.stopOnFailure (#18009)

docs/src/test-api/class-testproject.md

@@ -267,6 +267,13 @@ An integer number that defines when the project should run relative to other pro
 one stage. By default all projects run in stage 0. Stages with lower number run first. Several projects can run in
 each stage. Exeution order between projecs in the same stage is undefined.
 
+## property: TestProject.stopOnFailure
+* since: v1.28
+- type: ?<[boolean]>
+
+If set to true and the any test in the project fails all subsequent projects in the same playwright test run will
+be skipped.
+
 ## property: TestProject.testDir
 * since: v1.10
 - type: ?<[string]>

packages/playwright-test/src/dispatcher.ts

@@ -30,6 +30,7 @@ export type TestGroup = {
   requireFile: string;
   repeatEachIndex: number;
   projectId: string;
+  stopOnFailure: boolean;
   tests: TestCase[];
   watchMode: boolean;
 };

packages/playwright-test/src/loader.ts

@@ -278,6 +278,7 @@ export class Loader {
     const snapshotDir = takeFirst(projectConfig.snapshotDir, config.snapshotDir, testDir);
     const name = takeFirst(projectConfig.name, config.name, '');
     const stage =  takeFirst(projectConfig.stage, 0);
+    const stopOnFailure = takeFirst(projectConfig.stopOnFailure, false);
 
     let screenshotsDir = takeFirst((projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name));
     if (process.env.PLAYWRIGHT_DOCKER) {
@@ -298,6 +299,7 @@ export class Loader {
       name,
       testDir,
       stage,
+      stopOnFailure,
       _respectGitIgnore: respectGitIgnore,
       snapshotDir,
       _screenshotsDir: screenshotsDir,
@@ -609,6 +611,16 @@ function validateProject(file: string, project: Project, title: string) {
       throw errorWithFile(file, `${title}.retries must be a non-negative number`);
   }
 
+  if ('stage' in project && project.stage !== undefined) {
+    if (typeof project.stage !== 'number' || Math.floor(project.stage) !== project.stage)
+      throw errorWithFile(file, `${title}.stage must be an integer`);
+  }
+
+  if ('stopOnFailure' in project && project.stopOnFailure !== undefined) {
+    if (typeof project.stopOnFailure !== 'boolean')
+      throw errorWithFile(file, `${title}.stopOnFailure must be a boolean`);
+  }
+
   if ('testDir' in project && project.testDir !== undefined) {
     if (typeof project.testDir !== 'string')
       throw errorWithFile(file, `${title}.testDir must be a string`);

packages/playwright-test/src/runner.ts

@@ -426,7 +426,7 @@ export class Runner {
 
       let hasWorkerErrors = false;
       for (const testGroups of concurrentTestGroups) {
-        const dispatcher = new Dispatcher(this._loader, testGroups, this._reporter);
+        const dispatcher = new Dispatcher(this._loader, [...testGroups], this._reporter);
         sigintWatcher = new SigIntWatcher();
         await Promise.race([dispatcher.run(), sigintWatcher.promise()]);
         if (!sigintWatcher.hadSignal()) {
@@ -438,7 +438,8 @@ export class Runner {
         hasWorkerErrors = dispatcher.hasWorkerErrors();
         if (hasWorkerErrors)
           break;
-        if (testGroups.some(testGroup => testGroup.tests.some(test => !test.ok())))
+        const stopOnFailureGroups = testGroups.filter(group => group.stopOnFailure);
+        if (stopOnFailureGroups.some(testGroup => testGroup.tests.some(test => !test.ok())))
           break;
         if (sigintWatcher.hadSignal())
           break;
@@ -747,6 +748,7 @@ function createTestGroups(projectSuites: Suite[], workers: number): TestGroup[]
       requireFile: test._requireFile,
       repeatEachIndex: test.repeatEachIndex,
       projectId: test._projectId,
+      stopOnFailure: test.parent.project()!.stopOnFailure,
       tests: [],
       watchMode: false,
     };

packages/playwright-test/types/test.d.ts

@@ -262,6 +262,11 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
    * stage. Exeution order between projecs in the same stage is undefined.
    */
   stage: number;
+  /**
+   * If set to true and the any test in the project fails all subsequent projects in the same playwright test run will be
+   * skipped.
+   */
+  stopOnFailure: boolean;
   /**
    * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file.
    *
@@ -4471,6 +4476,12 @@ interface TestProject {
    */
   stage?: number;
 
+  /**
+   * If set to true and the any test in the project fails all subsequent projects in the same playwright test run will be
+   * skipped.
+   */
+  stopOnFailure?: boolean;
+
   /**
    * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file.
    *

tests/playwright-test/config.spec.ts

@@ -480,3 +480,60 @@ test('should have correct types for the config', async ({ runTSC }) => {
   });
   expect(result.exitCode).toBe(0);
 });
+
+test('should throw when project.stage is not a number', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          projects: [
+            { name: 'a', stage: 'foo' },
+          ],
+        };
+    `,
+    'a.test.ts': `
+        const { test } = pwt;
+        test('pass', async () => {});
+      `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`config.projects[0].stage must be an integer`);
+});
+
+test('should throw when project.stage is not an integer', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          projects: [
+            { name: 'a', stage: 3.14 },
+          ],
+        };
+    `,
+    'a.test.ts': `
+        const { test } = pwt;
+        test('pass', async () => {});
+      `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`config.projects[0].stage must be an integer`);
+});
+
+test('should throw when project.stopOnFailure is not a boolean', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          projects: [
+            { name: 'a', stopOnFailure: 'yes' },
+          ],
+        };
+    `,
+    'a.test.ts': `
+        const { test } = pwt;
+        test('pass', async () => {});
+      `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`config.projects[0].stopOnFailure must be a boolean`);
+});

tests/playwright-test/stages.spec.ts

@@ -207,3 +207,81 @@ test('should work with project filter', async ({ runGroups }, testInfo) => {
   expectRunBefore(timeline, ['c'], ['b']); // 0 < 10
   expect(passed).toBe(3);
 });
+
+test('should continue after failures', async ({ runGroups }, testInfo) => {
+  const projectTemplates = {
+    'a': {
+      stage: 1
+    },
+    'b': {
+      stage: 2
+    },
+    'c': {
+      stage: 2
+    },
+    'd': {
+      stage: 4
+    },
+    'e': {
+      stage: 4
+    },
+  };
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e'], testInfo, projectTemplates);
+  configWithFiles[`b/b.spec.ts`] = `
+    const { test } = pwt;
+    test('b test', async () => {
+      expect(1).toBe(2);
+    });`;
+  configWithFiles[`d/d.spec.ts`] = `
+    const { test } = pwt;
+    test('d test', async () => {
+      expect(1).toBe(2);
+    });`;
+  const { exitCode, passed, failed, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(1);
+  expect(failed).toBe(2);
+  expect(passed).toBe(3);
+  expect(projectNames(timeline)).toEqual(['a', 'b', 'c', 'd', 'e']);
+  expectRunBefore(timeline, ['a'], ['b', 'c', 'd', 'e']); // 1 < 2
+  expectRunBefore(timeline, ['b', 'c'], ['d', 'e']); // 2 < 4
+});
+
+test('should support stopOnFailire', async ({ runGroups }, testInfo) => {
+  const projectTemplates = {
+    'a': {
+      stage: 1
+    },
+    'b': {
+      stage: 2,
+      stopOnFailure: true
+    },
+    'c': {
+      stage: 2
+    },
+    'd': {
+      stage: 4,
+      stopOnFailure: true // this is not important as the test is skipped
+    },
+    'e': {
+      stage: 4
+    },
+  };
+  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e'], testInfo, projectTemplates);
+  configWithFiles[`b/b.spec.ts`] = `
+    const { test } = pwt;
+    test('b test', async () => {
+      expect(1).toBe(2);
+    });`;
+  configWithFiles[`d/d.spec.ts`] = `
+    const { test } = pwt;
+    test('d test', async () => {
+      expect(1).toBe(2);
+    });`;
+  const { exitCode, passed, failed, skipped, timeline } =  await runGroups(configWithFiles);
+  expect(exitCode).toBe(1);
+  expect(failed).toBe(1);
+  expect(passed).toBeLessThanOrEqual(2); // 'c' may either pass or be skipped.
+  expect(passed + skipped).toBe(4);
+  expect(projectNames(timeline)).not.toContainEqual(['d', 'e']);
+});
+

utils/generate_types/overrides-test.d.ts

@@ -47,6 +47,7 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
   repeatEach: number;
   retries: number;
   stage: number;
+  stopOnFailure: boolean;
   testDir: string;
   testIgnore: string | RegExp | (string | RegExp)[];
   testMatch: string | RegExp | (string | RegExp)[];