feat(testrunner): support repeat-each (#3652)

test-runner/src/cli.ts

@@ -38,6 +38,7 @@ program
     .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
     .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', String(Math.ceil(require('os').cpus().length / 2)))
     .option('--reporter <reporter>', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot')
+    .option('--repeat-each <repeat-each>', 'Specify how many times to run the tests', '1')
     .option('--retries <retries>', 'Specify retry count', '0')
     .option('--trial-run', 'Only collect the matching tests and report them as passing')
     .option('--quiet', 'Suppress stdio', false)
@@ -54,6 +55,7 @@ program
         grep: command.grep,
         jobs: parseInt(command.jobs, 10),
         outputDir: command.output,
+        repeatEach: parseInt(command.repeatEach, 10),
         retries: parseInt(command.retries, 10),
         snapshotDir: path.join(testDir, '__snapshots__'),
         testDir,

test-runner/src/index.ts

@@ -17,6 +17,8 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
+import rimraf from 'rimraf';
+import { promisify } from 'util';
 import './builtin.fixtures';
 import './expect';
 import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT, TestInfo } from './fixtures';
@@ -30,6 +32,8 @@ export { Reporter } from './reporter';
 export { RunnerConfig } from './runnerConfig';
 export { Suite, Test } from './test';
 
+const removeFolderAsync = promisify(rimraf);
+
 declare global {
   interface WorkerState {
   }
@@ -89,6 +93,7 @@ export async function run(config: RunnerConfig, files: string[], reporter: Repor
   // Trial run does not need many workers, use one.
   const jobs = (config.trialRun || config.debug) ? 1 : config.jobs;
   const runner = new Runner(suite, { ...config, jobs }, reporter);
+  await removeFolderAsync(config.outputDir).catch(e => {});
   fs.mkdirSync(config.outputDir, { recursive: true });
   try {
     for (const f of beforeFunctions)

test-runner/src/runnerConfig.ts

@@ -24,6 +24,7 @@ export type RunnerConfig = {
   debug?: boolean;
   quiet?: boolean;
   grep?: string;
+  repeatEach: number;
   retries: number,
   trialRun?: boolean;
   updateSnapshots?: boolean;

test-runner/src/testCollector.ts

@@ -20,6 +20,7 @@ import { Test, Suite, serializeConfiguration } from './test';
 import { spec } from './spec';
 import { RunnerConfig } from './runnerConfig';
 
+
 export type Matrix = {
   [key: string]: string[]
 };
@@ -93,14 +94,17 @@ export class TestCollector {
       }
     });
 
-    // Clone the suite as many times as there are worker hashes.
-    // Only include the tests that requested these generations.
-    for (const [hash, {configuration, configurationString, tests}] of workerGeneratorConfigurations.entries()) {
-      const clone = this._cloneSuite(suite, tests);
-      this.suite._addSuite(clone);
-      clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : '');
-      clone.configuration = configuration;
-      clone._configurationString = configurationString;
+    // Clone the suite as many times as we have repeat each.
+    for (let i = 0; i < this._config.repeatEach; ++i) {
+      // Clone the suite as many times as there are worker hashes.
+      // Only include the tests that requested these generations.
+      for (const [hash, {configuration, configurationString, tests}] of workerGeneratorConfigurations.entries()) {
+        const clone = this._cloneSuite(suite, tests);
+        this.suite._addSuite(clone);
+        clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : '') + ' ' + (i ? ` #repeat-${i}#` : '');
+        clone.configuration = configuration;
+        clone._configurationString = configurationString + `#repeat-${i}#`;
+      }
     }
   }
 

test-runner/test/exit-code.spec.ts

@@ -45,10 +45,9 @@ it('should access error in fixture', async () => {
 });
 
 it('should access data in fixture', async () => {
-  const result = await runTest('test-data-visible-in-fixture.js');
-  expect(result.exitCode).toBe(1);
-  const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'results.json')).toString());
-  const testResult = data.suites[0].tests[0].results[0];
+  const { exitCode, report } = await runTest('test-data-visible-in-fixture.js');
+  expect(exitCode).toBe(1);
+  const testResult = report.suites[0].tests[0].results[0];
   expect(testResult.data).toEqual({ 'myname': 'myvalue' });
   expect(testResult.stdout).toEqual([{ text: 'console.log\n' }]);
   expect(testResult.stderr).toEqual([{ text: 'console.error\n' }]);
@@ -67,10 +66,9 @@ it('should handle worker fixture error', async () => {
 });
 
 it('should collect stdio', async () => {
-  const result = await runTest('stdio.js');
-  expect(result.exitCode).toBe(0);
-  const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'results.json')).toString());
-  const testResult = data.suites[0].tests[0].results[0];
+  const { exitCode, report } = await runTest('stdio.js');
+  expect(exitCode).toBe(0);
+  const testResult = report.suites[0].tests[0].results[0];
   const { stdout, stderr } = testResult;
   expect(stdout).toEqual([{ text: 'stdout text' }, { buffer: Buffer.from('stdout buffer').toString('base64') }]);
   expect(stderr).toEqual([{ text: 'stderr text' }, { buffer: Buffer.from('stderr buffer').toString('base64') }]);
@@ -87,8 +85,17 @@ it('should retry failures', async () => {
   expect(result.flaky).toBe(1);
 });
 
+it('should repeat each', async () => {
+  const { exitCode, report } = await runTest('one-success.js', { 'repeat-each': 3 });
+  expect(exitCode).toBe(0);
+  expect(report.suites.length).toBe(3);
+  for (const suite of report.suites)
+    expect(suite.tests.length).toBe(1);
+});
+
 async function runTest(filePath: string, params: any = {}) {
   const outputDir = path.join(__dirname, 'test-results');
+  const reportFile = path.join(outputDir, 'results.json');
   await removeFolderAsync(outputDir).catch(e => {});
 
   const { output, status } = spawnSync('node', [
@@ -100,17 +107,19 @@ async function runTest(filePath: string, params: any = {}) {
   ], {
     env: {
       ...process.env,
-      PWRUNNER_JSON_REPORT: path.join(__dirname, 'test-results', 'results.json'),
+      PWRUNNER_JSON_REPORT: reportFile,
     }
   });
   const passed = (/(\d+) passed/.exec(output.toString()) || [])[1];
   const failed = (/(\d+) failed/.exec(output.toString()) || [])[1];
   const flaky = (/(\d+) flaky/.exec(output.toString()) || [])[1];
+  const report = JSON.parse(fs.readFileSync(reportFile).toString());
   return {
     exitCode: status,
     output: output.toString(),
     passed: parseInt(passed, 10),
     failed: parseInt(failed || '0', 10),
-    flaky: parseInt(flaky || '0', 10)
+    flaky: parseInt(flaky || '0', 10),
+    report
   };
 }