Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore(testrunner): extract runtime api and use it from cli (#3585)

Browse source
This commit is contained in:
Pavel Feldman 2020-08-23 08:22:14 -07:00 committed by GitHub
parent 2e1493a5fa
commit 224d3df899
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 131 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
test-runner/src/cli.ts
View file

@ -14,21 +14,11 @@
* limitations under the License.
*/
import program from 'commander';
import * as fs from 'fs';
import * as path from 'path';
import program from 'commander';
import { collectTests, runTests, RunnerConfig } from '.';
import { reporters } from './reporters';
import { installTransform } from './transform';
import { Runner } from './runner';
import { TestCollector } from './testCollector';
let beforeFunction;
let afterFunction;
let matrix = {};
global['before'] = (fn => beforeFunction = fn);
global['after'] = (fn => afterFunction = fn);
global['matrix'] = (m => matrix = m);
program
.version('Version ' + /** @type {any} */ (require)('../package.json').version)
@ -43,39 +33,32 @@ program
.option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
.option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
.action(async (command) => {
// Collect files]
const testDir = path.resolve(process.cwd(), command.args[0]);
const files = collectFiles(testDir, '', command.args.slice(1));
const revertBabelRequire = installTransform();
let hasSetup = false;
try {
hasSetup = fs.statSync(path.join(testDir, 'setup.js')).isFile();
} catch (e) {
}
try {
hasSetup = hasSetup || fs.statSync(path.join(testDir, 'setup.ts')).isFile();
} catch (e) {
}
if (hasSetup)
require(path.join(testDir, 'setup'));
revertBabelRequire();
const testCollector = new TestCollector(files, matrix, {
forbidOnly: command.forbidOnly || undefined,
const config: RunnerConfig = {
debug: command.debug,
quiet: command.quiet,
grep: command.grep,
jobs: command.jobs,
outputDir: command.output,
snapshotDir: path.join(testDir, '__snapshots__'),
testDir,
timeout: command.timeout,
});
const rootSuite = testCollector.suite;
if (command.forbidOnly && testCollector.hasOnly()) {
console.error('=====================================');
console.error(' --forbid-only found a focused test.');
console.error('=====================================');
process.exit(1);
trialRun: command.trialRun,
updateSnapshots: command.updateSnapshots
};
const files = collectFiles(testDir, '', command.args.slice(1));
const suite = collectTests(config, files);
if (command.forbidOnly) {
const hasOnly = suite.eachTest(t => t.only) || suite.eachSuite(s => s.only);
if (hasOnly) {
console.error('=====================================');
console.error(' --forbid-only found a focused test.');
console.error('=====================================');
process.exit(1);
}
}
const total = rootSuite.total();
const total = suite.total();
if (!total) {
console.error('=================');
console.error(' no tests found.');
@ -83,38 +66,15 @@ program
process.exit(1);
}
// Trial run does not need many workers, use one.
const jobs = (command.trialRun || command.debug) ? 1 : command.jobs;
const runner = new Runner(rootSuite, total, {
debug: command.debug,
quiet: command.quiet,
grep: command.grep,
jobs,
outputDir: command.output,
snapshotDir: path.join(testDir, '__snapshots__'),
testDir,
timeout: command.timeout,
trialRun: command.trialRun,
updateSnapshots: command.updateSnapshots
});
const reporterFactory = reporters[command.reporter || 'dot'];
new reporterFactory(runner);
try {
if (beforeFunction)
await beforeFunction();
await runner.run();
await runner.stop();
} finally {
if (afterFunction)
await afterFunction();
}
process.exit(runner.stats.failures ? 1 : 0);
await runTests(config, suite, reporterFactory);
const hasFailures = suite.eachTest(t => t.error);
process.exit(hasFailures ? 1 : 0);
});
program.parse(process.argv);
function collectFiles(testDir, dir, filters) {
function collectFiles(testDir: string, dir: string, filters: string[]): string[] {
const fullDir = path.join(testDir, dir);
if (fs.statSync(fullDir).isFile())
return [fullDir];

6
test-runner/src/fixturesUI.ts
View file

@ -51,6 +51,7 @@ function specBuilder(modifiers, specCallback) {
export function fixturesUI(suite: Suite, file: string, timeout: number): () => void {
const suites = [suite];
suite.file = file;
const it = specBuilder(['skip', 'fail', 'slow', 'only'], (specs, title, fn) => {
const suite = suites[0];
@ -66,14 +67,13 @@ export function fixturesUI(suite: Suite, file: string, timeout: number): () => v
test.pending = true;
if (!only && specs.fail && specs.fail[0])
test.pending = true;
test.pending = test.pending || suite.isPending();
suite.addTest(test);
suite._addTest(test);
return test;
});
const describe = specBuilder(['skip', 'fail', 'only'], (specs, title, fn) => {
const child = new Suite(title, suites[0]);
suites[0].addSuite(child);
suites[0]._addSuite(child);
child.file = file;
const only = specs.only && specs.only[0];
if (only)

54
test-runner/src/index.ts
View file

@ -15,12 +15,28 @@
* limitations under the License.
*/
import * as fs from 'fs';
import * as path from 'path';
import './builtin.fixtures';
import './expect';
import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT } from './fixtures';
import { reporters } from './reporters';
import { Runner } from './runner';
import { RunnerConfig } from './runnerConfig';
import { Test } from './test';
import { Suite, Test } from './test';
import { Matrix, TestCollector } from './testCollector';
import { installTransform } from './transform';
export { parameters, registerParameter } from './fixtures';
export { RunnerConfig } from './runnerConfig';
export { Suite, Test } from './test';
let beforeFunctions: Function[] = [];
let afterFunctions: Function[] = [];
let matrix: Matrix = {};
global['before'] = (fn: Function) => beforeFunctions.push(fn);
global['after'] = (fn: Function) => afterFunctions.push(fn);
global['matrix'] = (m: Matrix) => matrix = m;
export function registerFixture<T extends keyof TestState>(name: T, fn: (params: FixtureParameters & WorkerState & TestState, runTest: (arg: TestState[T]) => Promise<void>, config: RunnerConfig, test: Test) => Promise<void>) {
registerFixtureT<RunnerConfig, T>(name, fn);
@ -29,3 +45,39 @@ export function registerFixture<T extends keyof TestState>(name: T, fn: (params:
export function registerWorkerFixture<T extends keyof (WorkerState & FixtureParameters)>(name: T, fn: (params: FixtureParameters & WorkerState, runTest: (arg: (WorkerState & FixtureParameters)[T]) => Promise<void>, config: RunnerConfig) => Promise<void>) {
registerWorkerFixtureT<RunnerConfig, T>(name, fn);
};
export function collectTests(config: RunnerConfig, files: string[]): Suite {
const revertBabelRequire = installTransform();
let hasSetup = false;
try {
hasSetup = fs.statSync(path.join(config.testDir, 'setup.js')).isFile();
} catch (e) {
}
try {
hasSetup = hasSetup || fs.statSync(path.join(config.testDir, 'setup.ts')).isFile();
} catch (e) {
}
if (hasSetup)
require(path.join(config.testDir, 'setup'));
revertBabelRequire();
const testCollector = new TestCollector(files, matrix, config);
return testCollector.suite;
}
export async function runTests(config: RunnerConfig, suite: Suite, reporterFactory: any) {
// Trial run does not need many workers, use one.
const jobs = (config.trialRun || config.debug) ? 1 : config.jobs;
const runner = new Runner(suite, { ...config, jobs });
new reporterFactory(runner);
try {
for (const f of beforeFunctions)
await f();
await runner.run();
await runner.stop();
} finally {
for (const f of afterFunctions)
await f();
}
}

8
test-runner/src/reporters.ts
View file

@ -179,20 +179,22 @@ export class JSONReporter extends BaseReporter {
runner.once('end', () => {
const result = {
config: this.config,
tests: this.suite.tests.map(test => this._serializeTest(test)),
suites: this.suite.suites.map(suite => this._serializeSuite(suite))
suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s)
};
console.log(JSON.stringify(result, undefined, 2));
});
}
private _serializeSuite(suite: Suite): any {
if (!suite.eachTest(test => true))
return null;
const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s);
return {
title: suite.title,
file: suite.file,
configuration: suite.configuration,
tests: suite.tests.map(test => this._serializeTest(test)),
suites: suite.suites.map(suite => this._serializeSuite(suite))
suites: suites.length ? suites : undefined
};
}

3
test-runner/src/runner.ts
View file

@ -36,7 +36,7 @@ export class Runner extends EventEmitter {
readonly _config: RunnerConfig;
private _suite: Suite;
constructor(suite: Suite, total: number, config: RunnerConfig) {
constructor(suite: Suite, config: RunnerConfig) {
super();
this._config = config;
@ -68,6 +68,7 @@ export class Runner extends EventEmitter {
});
if (process.stdout.isTTY) {
const total = suite.total();
console.log();
const jobs = Math.min(config.jobs, this._testsByConfiguredFile.size);
console.log(`Running ${total} test${ total > 1 ? 's' : '' } using ${jobs} worker${ jobs > 1 ? 's' : ''}`);

36
test-runner/src/test.ts
View file

@ -39,7 +39,15 @@ export class Test {
this.fn = fn;
}
clone(): Test {
titlePath(): string[] {
return [...this.suite.titlePath(), this.title];
}
fullTitle(): string {
return this.titlePath().join(' ');
}
_clone(): Test {
const test = new Test(this.title, this.fn);
test.suite = this.suite;
test.only = this.only;
@ -49,14 +57,6 @@ export class Test {
test._overriddenFn = this._overriddenFn;
return test;
}
titlePath(): string[] {
return [...this.suite.titlePath(), this.title];
}
fullTitle(): string {
return this.titlePath().join(' ');
}
}
export class Suite {
@ -91,22 +91,30 @@ export class Suite {
return count;
}
isPending(): boolean {
return this.pending || (this.parent && this.parent.isPending());
_isPending(): boolean {
return this.pending || (this.parent && this.parent._isPending());
}
addTest(test: Test) {
_addTest(test: Test) {
test.suite = this;
this.tests.push(test);
this._entries.push(test);
}
addSuite(suite: Suite) {
_addSuite(suite: Suite) {
suite.parent = this;
this.suites.push(suite);
this._entries.push(suite);
}
eachSuite(fn: (suite: Suite) => boolean | void): boolean {
for (const suite of this.suites) {
if (suite.eachSuite(fn))
return true;
}
return false;
}
eachTest(fn: (test: Test) => boolean | void): boolean {
for (const suite of this.suites) {
if (suite.eachTest(fn))
@ -119,7 +127,7 @@ export class Suite {
return false;
}
clone(): Suite {
_clone(): Suite {
const suite = new Suite(this.title);
suite.only = this.only;
suite.file = this.file;

31
test-runner/src/testCollector.ts
View file

@ -18,21 +18,26 @@ import path from 'path';
import { fixturesForCallback } from './fixtures';
import { Configuration, Test, Suite } from './test';
import { fixturesUI } from './fixturesUI';
import { RunnerConfig } from './runnerConfig';
export type Matrix = {
[key: string]: string[]
};
export class TestCollector {
suite: Suite;
private _matrix: { [key: string]: string; };
private _options: any;
private _matrix: Matrix;
private _config: RunnerConfig;
private _grep: RegExp;
private _hasOnly: boolean;
constructor(files: string[], matrix: { [key: string] : string }, options) {
constructor(files: string[], matrix: Matrix, config: RunnerConfig) {
this._matrix = matrix;
this._options = options;
this._config = config;
this.suite = new Suite('');
if (options.grep) {
const match = options.grep.match(/^\/(.*)\/(g|i|)$|.*/);
if (config.grep) {
const match = config.grep.match(/^\/(.*)\/(g|i|)$|.*/);
this._grep = new RegExp(match[1] || match[0], match[2]);
}
@ -46,9 +51,9 @@ export class TestCollector {
return this._hasOnly;
}
_addFile(file) {
_addFile(file: string) {
const suite = new Suite('');
const revertBabelRequire = fixturesUI(suite, file, this._options.timeout);
const revertBabelRequire = fixturesUI(suite, file, this._config.timeout);
require(file);
revertBabelRequire();
suite._renumber();
@ -95,30 +100,30 @@ export class TestCollector {
// Only include the tests that requested these generations.
for (const [hash, {configurationObject, configurationString, tests}] of workerGeneratorConfigurations.entries()) {
const clone = this._cloneSuite(suite, configurationObject, configurationString, tests);
this.suite.addSuite(clone);
this.suite._addSuite(clone);
clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : '');
}
}
_cloneSuite(suite: Suite, configurationObject: Configuration, configurationString: string, tests: Set<Test>) {
const copy = suite.clone();
const copy = suite._clone();
copy.only = suite.only;
copy.configuration = configurationObject;
for (const entry of suite._entries) {
if (entry instanceof Suite) {
copy.addSuite(this._cloneSuite(entry, configurationObject, configurationString, tests));
copy._addSuite(this._cloneSuite(entry, configurationObject, configurationString, tests));
} else {
const test = entry;
if (!tests.has(test))
continue;
if (this._grep && !this._grep.test(test.fullTitle()))
continue;
const testCopy = test.clone();
const testCopy = test._clone();
testCopy.only = test.only;
testCopy._ordinal = test._ordinal;
testCopy._configurationObject = configurationObject;
testCopy._configurationString = configurationString;
copy.addTest(testCopy);
copy._addTest(testCopy);
}
}
return copy;

2
test-runner/src/testRunner.ts
View file

@ -107,7 +107,7 @@ export class TestRunner extends EventEmitter {
if (this._ordinals.size && !this._ordinals.has(ordinal))
return;
this._remaining.delete(ordinal);
if (test.pending) {
if (test.pending || test.suite._isPending()) {
this.emit('pending', { test: this._serializeTest(test) });
return;
}