test: expose repeatEachIndex (#11158)

.github/workflows/tests_primary.yml

@@ -9,7 +9,6 @@ on:
     paths-ignore:
       - 'browser_patches/**'
       - 'docs/**'
-    types: [ labeled ]
     branches:
       - main
       - release-*

docs/src/test-reporter-api/class-testcase.md

@@ -46,6 +46,11 @@ Testing outcome for this test. Note that outcome is not the same as [`property:
 
 Suite this test case belongs to.
 
+## property: TestCase.repeatEachIndex
+- type: <[int]>
+
+Contains the repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to the [command line](./test-cli.md).
+
 ## property: TestCase.results
 - type: <[Array]<[TestResult]>>
 

packages/playwright-test/src/project.ts

@@ -79,7 +79,7 @@ export class ProjectImpl {
         const test = entry._clone();
         test.retries = this.config.retries;
         test._id = `${entry._ordinalInFile}@${entry._requireFile}#run${this.index}-repeat${repeatEachIndex}`;
-        test._repeatEachIndex = repeatEachIndex;
+        test.repeatEachIndex = repeatEachIndex;
         test._projectIndex = this.index;
         to._addTest(test);
         if (!filter(test)) {
@@ -100,7 +100,7 @@ export class ProjectImpl {
       clone._pool = this.buildPool(hook);
       clone._projectIndex = this.index;
       clone._id = `${hook._ordinalInFile}@${hook._requireFile}#run${this.index}-repeat${repeatEachIndex}`;
-      clone._repeatEachIndex = repeatEachIndex;
+      clone.repeatEachIndex = repeatEachIndex;
       to._addAllHook(clone);
     }
     return true;

packages/playwright-test/src/reporters/raw.ts

@@ -181,8 +181,8 @@ class RawReporter {
   }
 
   private _serializeSuite(suite: Suite): JsonSuite {
-    const fileId = calculateSha1(suite.location!.file.split(path.sep).join('/'));
     const location = this._relativeLocation(suite.location);
+    const fileId = calculateSha1(location!.file.split(path.sep).join('/'));
     return {
       title: suite.title,
       fileId,
@@ -195,7 +195,7 @@ class RawReporter {
 
   private _serializeTest(test: TestCase, fileId: string): JsonTestCase {
     const [, projectName, , ...titles] = test.titlePath();
-    const testIdExpression = `project:${projectName}|path:${titles.join('>')}`;
+    const testIdExpression = `project:${projectName}|path:${titles.join('>')}|repeat:${test.repeatEachIndex}`;
     const testId = fileId + '-' + calculateSha1(testIdExpression);
     return {
       testId,

packages/playwright-test/src/runner.ts

@@ -475,7 +475,7 @@ function createTestGroups(rootSuite: Suite): TestGroup[] {
     return {
       workerHash: test._workerHash,
       requireFile: test._requireFile,
-      repeatEachIndex: test._repeatEachIndex,
+      repeatEachIndex: test.repeatEachIndex,
       projectIndex: test._projectIndex,
       tests: [],
     };

packages/playwright-test/src/test.ts

@@ -131,6 +131,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
   timeout = 0;
   annotations: Annotations = [];
   retries = 0;
+  repeatEachIndex = 0;
 
   _type: TestCaseType;
   _ordinalInFile: number;
@@ -138,7 +139,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {
   _id = '';
   _workerHash = '';
   _pool: FixturePool | undefined;
-  _repeatEachIndex = 0;
   _projectIndex = 0;
 
   constructor(type: TestCaseType, title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl, location: Location) {

packages/playwright-test/types/testReporter.d.ts

@@ -166,6 +166,11 @@ export interface TestCase {
    * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries).
    */
   retries: number;
+  /**
+   * Contains the repeat index when running in "repeat each" mode. This mode is enabled by passing `--repeat-each` to the
+   * [command line](https://playwright.dev/docs/test-cli).
+   */
+  repeatEachIndex: number;
   /**
    * Results for each run of this test.
    */

tests/playwright-test/reporter-html.spec.ts

@@ -424,3 +424,26 @@ test('should strikethough textual diff with commonalities', async ({ runInlineTe
   const stricken = await page.locator('css=strike').innerText();
   expect(stricken).toBe('old');
 });
+
+test('should differentiate repeat-each test cases', async ({ runInlineTest, showReport, page }) => {
+  test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/10859' });
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = pwt;
+      test('sample', async ({}, testInfo) => {
+        if (testInfo.repeatEachIndex === 2)
+          throw new Error('ouch');
+      });
+    `
+  }, { 'reporter': 'dot,html', 'repeat-each': 3 });
+  expect(result.exitCode).toBe(1);
+  await showReport();
+
+  await page.locator('text=sample').first().click();
+  await expect(page.locator('text=ouch')).toBeVisible();
+  await page.locator('text=All').first().click();
+
+  await page.locator('text=sample').nth(1).click();
+  await expect(page.locator('text=Before Hooks')).toBeVisible();
+  await expect(page.locator('text=ouch')).toBeHidden();
+});

utils/generate_types/overrides-testReporter.d.ts

@@ -44,6 +44,7 @@ export interface TestCase {
   timeout: number;
   annotations: { type: string, description?: string }[];
   retries: number;
+  repeatEachIndex: number;
   results: TestResult[];
   outcome(): 'skipped' | 'expected' | 'unexpected' | 'flaky';
   ok(): boolean;