From 06154060a20568f07c8963b67f98f284fd006f88 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 26 Aug 2020 20:18:44 -0700 Subject: [PATCH] feat(testrunner): support repeat-each (#3652) --- test-runner/src/cli.ts | 2 ++ test-runner/src/index.ts | 5 +++++ test-runner/src/runnerConfig.ts | 1 + test-runner/src/testCollector.ts | 20 ++++++++++++-------- test-runner/test/exit-code.spec.ts | 29 +++++++++++++++++++---------- 5 files changed, 39 insertions(+), 18 deletions(-) diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts index 606b8d74c2..6f102ea950 100644 --- a/test-runner/src/cli.ts +++ b/test-runner/src/cli.ts @@ -38,6 +38,7 @@ program .option('-g, --grep ', 'Only run tests matching this string or regexp', '.*') .option('-j, --jobs ', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', String(Math.ceil(require('os').cpus().length / 2))) .option('--reporter ', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot') + .option('--repeat-each ', 'Specify how many times to run the tests', '1') .option('--retries ', 'Specify retry count', '0') .option('--trial-run', 'Only collect the matching tests and report them as passing') .option('--quiet', 'Suppress stdio', false) @@ -54,6 +55,7 @@ program grep: command.grep, jobs: parseInt(command.jobs, 10), outputDir: command.output, + repeatEach: parseInt(command.repeatEach, 10), retries: parseInt(command.retries, 10), snapshotDir: path.join(testDir, '__snapshots__'), testDir, diff --git a/test-runner/src/index.ts b/test-runner/src/index.ts index c0bda73a03..e9665b290b 100644 --- a/test-runner/src/index.ts +++ b/test-runner/src/index.ts @@ -17,6 +17,8 @@ import * as fs from 'fs'; import * as path from 'path'; +import rimraf from 'rimraf'; +import { promisify } from 'util'; import './builtin.fixtures'; import './expect'; import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT, TestInfo } from './fixtures'; @@ -30,6 +32,8 @@ export { Reporter } from './reporter'; export { RunnerConfig } from './runnerConfig'; export { Suite, Test } from './test'; +const removeFolderAsync = promisify(rimraf); + declare global { interface WorkerState { } @@ -89,6 +93,7 @@ export async function run(config: RunnerConfig, files: string[], reporter: Repor // Trial run does not need many workers, use one. const jobs = (config.trialRun || config.debug) ? 1 : config.jobs; const runner = new Runner(suite, { ...config, jobs }, reporter); + await removeFolderAsync(config.outputDir).catch(e => {}); fs.mkdirSync(config.outputDir, { recursive: true }); try { for (const f of beforeFunctions) diff --git a/test-runner/src/runnerConfig.ts b/test-runner/src/runnerConfig.ts index 1be76f2bf0..6d585067e0 100644 --- a/test-runner/src/runnerConfig.ts +++ b/test-runner/src/runnerConfig.ts @@ -24,6 +24,7 @@ export type RunnerConfig = { debug?: boolean; quiet?: boolean; grep?: string; + repeatEach: number; retries: number, trialRun?: boolean; updateSnapshots?: boolean; diff --git a/test-runner/src/testCollector.ts b/test-runner/src/testCollector.ts index b83b4926d6..be8d82eb82 100644 --- a/test-runner/src/testCollector.ts +++ b/test-runner/src/testCollector.ts @@ -20,6 +20,7 @@ import { Test, Suite, serializeConfiguration } from './test'; import { spec } from './spec'; import { RunnerConfig } from './runnerConfig'; + export type Matrix = { [key: string]: string[] }; @@ -93,14 +94,17 @@ export class TestCollector { } }); - // Clone the suite as many times as there are worker hashes. - // Only include the tests that requested these generations. - for (const [hash, {configuration, configurationString, tests}] of workerGeneratorConfigurations.entries()) { - const clone = this._cloneSuite(suite, tests); - this.suite._addSuite(clone); - clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : ''); - clone.configuration = configuration; - clone._configurationString = configurationString; + // Clone the suite as many times as we have repeat each. + for (let i = 0; i < this._config.repeatEach; ++i) { + // Clone the suite as many times as there are worker hashes. + // Only include the tests that requested these generations. + for (const [hash, {configuration, configurationString, tests}] of workerGeneratorConfigurations.entries()) { + const clone = this._cloneSuite(suite, tests); + this.suite._addSuite(clone); + clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : '') + ' ' + (i ? ` #repeat-${i}#` : ''); + clone.configuration = configuration; + clone._configurationString = configurationString + `#repeat-${i}#`; + } } } diff --git a/test-runner/test/exit-code.spec.ts b/test-runner/test/exit-code.spec.ts index 50f182892a..ded73a2e3c 100644 --- a/test-runner/test/exit-code.spec.ts +++ b/test-runner/test/exit-code.spec.ts @@ -45,10 +45,9 @@ it('should access error in fixture', async () => { }); it('should access data in fixture', async () => { - const result = await runTest('test-data-visible-in-fixture.js'); - expect(result.exitCode).toBe(1); - const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'results.json')).toString()); - const testResult = data.suites[0].tests[0].results[0]; + const { exitCode, report } = await runTest('test-data-visible-in-fixture.js'); + expect(exitCode).toBe(1); + const testResult = report.suites[0].tests[0].results[0]; expect(testResult.data).toEqual({ 'myname': 'myvalue' }); expect(testResult.stdout).toEqual([{ text: 'console.log\n' }]); expect(testResult.stderr).toEqual([{ text: 'console.error\n' }]); @@ -67,10 +66,9 @@ it('should handle worker fixture error', async () => { }); it('should collect stdio', async () => { - const result = await runTest('stdio.js'); - expect(result.exitCode).toBe(0); - const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'results.json')).toString()); - const testResult = data.suites[0].tests[0].results[0]; + const { exitCode, report } = await runTest('stdio.js'); + expect(exitCode).toBe(0); + const testResult = report.suites[0].tests[0].results[0]; const { stdout, stderr } = testResult; expect(stdout).toEqual([{ text: 'stdout text' }, { buffer: Buffer.from('stdout buffer').toString('base64') }]); expect(stderr).toEqual([{ text: 'stderr text' }, { buffer: Buffer.from('stderr buffer').toString('base64') }]); @@ -87,8 +85,17 @@ it('should retry failures', async () => { expect(result.flaky).toBe(1); }); +it('should repeat each', async () => { + const { exitCode, report } = await runTest('one-success.js', { 'repeat-each': 3 }); + expect(exitCode).toBe(0); + expect(report.suites.length).toBe(3); + for (const suite of report.suites) + expect(suite.tests.length).toBe(1); +}); + async function runTest(filePath: string, params: any = {}) { const outputDir = path.join(__dirname, 'test-results'); + const reportFile = path.join(outputDir, 'results.json'); await removeFolderAsync(outputDir).catch(e => {}); const { output, status } = spawnSync('node', [ @@ -100,17 +107,19 @@ async function runTest(filePath: string, params: any = {}) { ], { env: { ...process.env, - PWRUNNER_JSON_REPORT: path.join(__dirname, 'test-results', 'results.json'), + PWRUNNER_JSON_REPORT: reportFile, } }); const passed = (/(\d+) passed/.exec(output.toString()) || [])[1]; const failed = (/(\d+) failed/.exec(output.toString()) || [])[1]; const flaky = (/(\d+) flaky/.exec(output.toString()) || [])[1]; + const report = JSON.parse(fs.readFileSync(reportFile).toString()); return { exitCode: status, output: output.toString(), passed: parseInt(passed, 10), failed: parseInt(failed || '0', 10), - flaky: parseInt(flaky || '0', 10) + flaky: parseInt(flaky || '0', 10), + report }; }