Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore(test-runner): remove the notion of Spec (#7661)

Browse source 
We now have Suites and Tests. When running multiple projects the whole
suite is cloned for each project. Same happens for repeatEach.

This simplifies the reporters API, but there is still room for improvement.

JSON reporter continues to produce old json output.
This commit is contained in:
Dmitry Gozman 2021-07-15 22:02:10 -07:00 committed by GitHub
parent 14b160d438
commit 8b2dd2e3d1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 418 additions and 334 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
src/test/dispatcher.ts
View file

@ -50,10 +50,8 @@ export class Dispatcher {
this._suite = suite;
for (const suite of this._suite.suites) {
for (const spec of suite._allSpecs()) {
for (const test of spec.tests)
this._testById.set(test._id, { test, result: test._appendTestResult() });
}
for (const test of suite._allTests())
this._testById.set(test._id, { test, result: test._appendTestResult() });
}
this._queue = this._filesSortedByWorkerHash();
@ -83,31 +81,29 @@ export class Dispatcher {
const entriesByWorkerHashAndFile = new Map<string, Map<string, DispatcherEntry>>();
for (const fileSuite of this._suite.suites) {
const file = fileSuite._requireFile;
for (const spec of fileSuite._allSpecs()) {
for (const test of spec.tests) {
let entriesByFile = entriesByWorkerHashAndFile.get(test._workerHash);
if (!entriesByFile) {
entriesByFile = new Map();
entriesByWorkerHashAndFile.set(test._workerHash, entriesByFile);
}
let entry = entriesByFile.get(file);
if (!entry) {
entry = {
runPayload: {
entries: [],
file,
},
repeatEachIndex: test._repeatEachIndex,
projectIndex: test._projectIndex,
hash: test._workerHash,
};
entriesByFile.set(file, entry);
}
entry.runPayload.entries.push({
retry: this._testById.get(test._id)!.result.retry,
testId: test._id,
});
for (const test of fileSuite._allTests()) {
let entriesByFile = entriesByWorkerHashAndFile.get(test._workerHash);
if (!entriesByFile) {
entriesByFile = new Map();
entriesByWorkerHashAndFile.set(test._workerHash, entriesByFile);
}
let entry = entriesByFile.get(file);
if (!entry) {
entry = {
runPayload: {
entries: [],
file,
},
repeatEachIndex: fileSuite._repeatEachIndex,
projectIndex: fileSuite._projectIndex,
hash: test._workerHash,
};
entriesByFile.set(file, entry);
}
entry.runPayload.entries.push({
retry: this._testById.get(test._id)!.result.retry,
testId: test._id,
});
}
}

63
src/test/project.ts
View file

@ -15,7 +15,7 @@
*/
import type { TestType, FullProject, Fixtures, FixturesWithLocation } from './types';
import { Spec, Test } from './test';
import { Suite, Test } from './test';
import { FixturePool } from './fixtures';
import { DeclaredFixtures, TestTypeImpl } from './testType';
@ -24,7 +24,7 @@ export class ProjectImpl {
private index: number;
private defines = new Map<TestType<any, any>, Fixtures>();
private testTypePools = new Map<TestTypeImpl, FixturePool>();
private specPools = new Map<Spec, FixturePool>();
private testPools = new Map<Test, FixturePool>();
constructor(project: FullProject, index: number) {
this.config = project;
@ -52,51 +52,60 @@ export class ProjectImpl {
return this.testTypePools.get(testType)!;
}
buildPool(spec: Spec): FixturePool {
if (!this.specPools.has(spec)) {
let pool = this.buildTestTypePool(spec._testType);
const overrides: Fixtures = spec.parent!._buildFixtureOverrides();
// TODO: we can optimize this function by building the pool inline in cloneSuite
private buildPool(test: Test): FixturePool {
if (!this.testPools.has(test)) {
let pool = this.buildTestTypePool(test._testType);
const overrides: Fixtures = test.parent!._buildFixtureOverrides();
if (Object.entries(overrides).length) {
const overridesWithLocation = {
fixtures: overrides,
location: {
file: spec.file,
file: test.file,
line: 1, // TODO: capture location
column: 1, // TODO: capture location
}
};
pool = new FixturePool([overridesWithLocation], pool);
}
this.specPools.set(spec, pool);
this.testPools.set(test, pool);
pool.validateFunction(spec.fn, 'Test', true, spec);
for (let parent = spec.parent; parent; parent = parent.parent) {
pool.validateFunction(test.fn, 'Test', true, test);
for (let parent = test.parent; parent; parent = parent.parent) {
for (const hook of parent._hooks)
pool.validateFunction(hook.fn, hook.type + ' hook', hook.type === 'beforeEach' || hook.type === 'afterEach', hook.location);
for (const modifier of parent._modifiers)
pool.validateFunction(modifier.fn, modifier.type + ' modifier', true, modifier.location);
}
}
return this.specPools.get(spec)!;
return this.testPools.get(test)!;
}
generateTests(spec: Spec, repeatEachIndex?: number) {
const digest = this.buildPool(spec).digest;
const min = repeatEachIndex === undefined ? 0 : repeatEachIndex;
const max = repeatEachIndex === undefined ? this.config.repeatEach - 1 : repeatEachIndex;
const tests: Test[] = [];
for (let i = min; i <= max; i++) {
const test = new Test(spec);
test.projectName = this.config.name;
test.retries = this.config.retries;
test._repeatEachIndex = i;
test._projectIndex = this.index;
test._workerHash = `run${this.index}-${digest}-repeat${i}`;
test._id = `${spec._ordinalInFile}@${spec._requireFile}#run${this.index}-repeat${i}`;
spec.tests.push(test);
tests.push(test);
cloneSuite(suite: Suite, repeatEachIndex: number, filter: (test: Test) => boolean): Suite | undefined {
const result = suite._clone();
result._repeatEachIndex = repeatEachIndex;
result._projectIndex = this.index;
for (const entry of suite._entries) {
if (entry instanceof Suite) {
const cloned = this.cloneSuite(entry, repeatEachIndex, filter);
if (cloned)
result._addSuite(cloned);
} else {
const pool = this.buildPool(entry);
const test = entry._clone();
test.projectName = this.config.name;
test.retries = this.config.retries;
test._workerHash = `run${this.index}-${pool.digest}-repeat${repeatEachIndex}`;
test._id = `${entry._ordinalInFile}@${entry._requireFile}#run${this.index}-repeat${repeatEachIndex}`;
test._pool = pool;
test._buildFullTitle(suite.fullTitle());
if (!filter(test))
continue;
result._addTest(test);
}
}
return tests;
if (result._entries.length)
return result;
}
private resolveFixtures(testType: TestTypeImpl): FixturesWithLocation[] {

15
src/test/reporter.ts
View file

@ -23,23 +23,16 @@ export interface Suite {
line: number;
column: number;
suites: Suite[];
specs: Spec[];
tests: Test[];
findTest(fn: (test: Test) => boolean | void): boolean;
findSpec(fn: (spec: Spec) => boolean | void): boolean;
totalTestCount(): number;
}
export interface Spec {
export interface Test {
suite: Suite;
title: string;
file: string;
line: number;
column: number;
tests: Test[];
fullTitle(): string;
ok(): boolean;
}
export interface Test {
spec: Spec;
results: TestResult[];
skipped: boolean;
expectedStatus: TestStatus;
@ -61,9 +54,9 @@ export interface TestResult {
stdout: (string | Buffer)[];
stderr: (string | Buffer)[];
}
export type FullResult = {
export interface FullResult {
status: 'passed' | 'failed' | 'timedout' | 'interrupted';
};
}
export interface Reporter {
onBegin(config: FullConfig, suite: Suite): void;
onTestBegin(test: Test): void;

17
src/test/reporters/base.ts
View file

@ -21,7 +21,7 @@ import fs from 'fs';
import milliseconds from 'ms';
import path from 'path';
import StackUtils from 'stack-utils';
import { FullConfig, TestStatus, Test, Spec, Suite, TestResult, TestError, Reporter, FullResult } from '../reporter';
import { FullConfig, TestStatus, Test, Suite, TestResult, TestError, Reporter, FullResult } from '../reporter';
const stackUtils = new StackUtils();
@ -56,7 +56,7 @@ export class BaseReporter implements Reporter {
}
onTestEnd(test: Test, result: TestResult) {
const relativePath = relativeSpecPath(this.config, test.spec);
const relativePath = relativeTestPath(this.config, test);
const fileAndProject = relativePath + (test.projectName ? ` [${test.projectName}]` : '');
const duration = this.fileDurations.get(fileAndProject) || 0;
this.fileDurations.set(fileAndProject, duration + result.duration);
@ -157,14 +157,13 @@ export function formatFailure(config: FullConfig, test: Test, index?: number): s
return tokens.join('\n');
}
function relativeSpecPath(config: FullConfig, spec: Spec): string {
return path.relative(config.rootDir, spec.file) || path.basename(spec.file);
function relativeTestPath(config: FullConfig, test: Test): string {
return path.relative(config.rootDir, test.file) || path.basename(test.file);
}
export function formatTestTitle(config: FullConfig, test: Test): string {
const spec = test.spec;
let relativePath = relativeSpecPath(config, spec);
relativePath += ':' + spec.line + ':' + spec.column;
let relativePath = relativeTestPath(config, test);
relativePath += ':' + test.line + ':' + test.column;
return `${relativePath} › ${test.fullTitle()}`;
}
@ -183,9 +182,9 @@ function formatFailedResult(test: Test, result: TestResult): string {
tokens.push('');
tokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), ' '));
if (result.error !== undefined)
tokens.push(indent(formatError(result.error, test.spec.file), ' '));
tokens.push(indent(formatError(result.error, test.file), ' '));
} else {
tokens.push(indent(formatError(result.error!, test.spec.file), ' '));
tokens.push(indent(formatError(result.error!, test.file), ' '));
}
return tokens.join('\n');
}

63
src/test/reporters/json.ts
View file

@ -17,7 +17,7 @@
import fs from 'fs';
import path from 'path';
import EmptyReporter from './empty';
import { FullConfig, Test, Suite, Spec, TestResult, TestError, FullResult, TestStatus } from '../reporter';
import { FullConfig, Test, Suite, TestResult, TestError, FullResult, TestStatus } from '../reporter';
export interface JSONReport {
config: Omit<FullConfig, 'projects'> & {
@ -119,13 +119,54 @@ class JSONReporter extends EmptyReporter {
};
})
},
suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[],
suites: this._mergeSuites(this.suite.suites),
errors: this._errors
};
}
private _mergeSuites(suites: Suite[]): JSONReportSuite[] {
debugger;
const fileSuites = new Map<string, JSONReportSuite>();
const result: JSONReportSuite[] = [];
for (const suite of suites) {
if (!fileSuites.has(suite.file)) {
const serialized = this._serializeSuite(suite);
if (serialized) {
fileSuites.set(suite.file, serialized);
result.push(serialized);
}
} else {
this._mergeTestsFromSuite(fileSuites.get(suite.file)!, suite);
}
}
return result;
}
private _mergeTestsFromSuite(to: JSONReportSuite, from: Suite) {
for (const fromSuite of from.suites) {
const toSuite = (to.suites || []).find(s => s.title === fromSuite.title && s.file === toPosixPath(path.relative(this.config.rootDir, fromSuite.file)) && s.line === fromSuite.line && s.column === fromSuite.column);
if (toSuite) {
this._mergeTestsFromSuite(toSuite, fromSuite);
} else {
const serialized = this._serializeSuite(fromSuite);
if (serialized) {
if (!to.suites)
to.suites = [];
to.suites.push(serialized);
}
}
}
for (const test of from.tests) {
const toSpec = to.specs.find(s => s.title === test.title && s.file === toPosixPath(path.relative(this.config.rootDir, test.file)) && s.line === test.line && s.column === test.column);
if (toSpec)
toSpec.tests.push(this._serializeTest(test));
else
to.specs.push(this._serializeTestSpec(test));
}
}
private _serializeSuite(suite: Suite): null | JSONReportSuite {
if (!suite.findSpec(test => true))
if (!suite.findTest(test => true))
return null;
const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[];
return {
@ -133,19 +174,19 @@ class JSONReporter extends EmptyReporter {
file: toPosixPath(path.relative(this.config.rootDir, suite.file)),
line: suite.line,
column: suite.column,
specs: suite.specs.map(test => this._serializeTestSpec(test)),
specs: suite.tests.map(test => this._serializeTestSpec(test)),
suites: suites.length ? suites : undefined,
};
}
private _serializeTestSpec(spec: Spec): JSONReportSpec {
private _serializeTestSpec(test: Test): JSONReportSpec {
return {
title: spec.title,
ok: spec.ok(),
tests: spec.tests.map(r => this._serializeTest(r)),
file: toPosixPath(path.relative(this.config.rootDir, spec.file)),
line: spec.line,
column: spec.column,
title: test.title,
ok: test.ok(),
tests: [ this._serializeTest(test) ],
file: toPosixPath(path.relative(this.config.rootDir, test.file)),
line: test.line,
column: test.column,
};
}

4
src/test/reporters/junit.ts
View file

@ -121,7 +121,7 @@ class JUnitReporter extends EmptyReporter {
const entry = {
name: 'testcase',
attributes: {
name: test.spec.fullTitle(),
name: test.fullTitle(),
classname: formatTestTitle(this.config, test),
time: (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000
},
@ -138,7 +138,7 @@ class JUnitReporter extends EmptyReporter {
entry.children.push({
name: 'failure',
attributes: {
message: `${path.basename(test.spec.file)}:${test.spec.line}:${test.spec.column} ${test.spec.title}`,
message: `${path.basename(test.file)}:${test.line}:${test.column} ${test.title}`,
type: 'FAILURE',
},
text: stripAscii(formatFailure(this.config, test))

112
src/test/runner.ts
View file

@ -22,7 +22,7 @@ import * as path from 'path';
import { promisify } from 'util';
import { Dispatcher } from './dispatcher';
import { createMatcher, FilePatternFilter, monotonicTime, raceAgainstDeadline } from './util';
import { Spec, Suite } from './test';
import { Test, Suite } from './test';
import { Loader } from './loader';
import { Reporter } from './reporter';
import { Multiplexer } from './reporters/multiplexer';
@ -41,16 +41,16 @@ const removeFolderAsync = promisify(rimraf);
const readDirAsync = promisify(fs.readdir);
const readFileAsync = promisify(fs.readFile);
type RunResultStatus = 'passed' | 'failed' | 'sigint' | 'forbid-only' | 'clashing-spec-titles' | 'no-tests' | 'timedout';
type RunResultStatus = 'passed' | 'failed' | 'sigint' | 'forbid-only' | 'clashing-test-titles' | 'no-tests' | 'timedout';
type RunResult = {
status: Exclude<RunResultStatus, 'forbid-only' | 'clashing-spec-titles'>;
status: Exclude<RunResultStatus, 'forbid-only' | 'clashing-test-titles'>;
} | {
status: 'forbid-only',
locations: string[]
} | {
status: 'clashing-spec-titles',
clashingSpecs: Map<string, Spec[]>
status: 'clashing-test-titles',
clashingTests: Map<string, Test[]>
};
export class Runner {
@ -114,13 +114,13 @@ export class Runner {
console.error('=================');
console.error(' no tests found.');
console.error('=================');
} else if (result?.status === 'clashing-spec-titles') {
} else if (result?.status === 'clashing-test-titles') {
console.error('=================');
console.error(' duplicate test titles are not allowed.');
for (const [title, specs] of result?.clashingSpecs.entries()) {
for (const [title, tests] of result?.clashingTests.entries()) {
console.error(` - title: ${title}`);
for (const spec of specs)
console.error(` - ${buildItemLocation(config.rootDir, spec)}`);
for (const test of tests)
console.error(` - ${buildItemLocation(config.rootDir, test)}`);
console.error('=================');
}
}
@ -175,38 +175,42 @@ export class Runner {
for (const file of allTestFiles)
await this._loader.loadTestFile(file);
const rootSuite = new Suite('');
const preprocessRoot = new Suite('');
for (const fileSuite of this._loader.fileSuites().values())
rootSuite._addSuite(fileSuite);
preprocessRoot._addSuite(fileSuite);
if (config.forbidOnly) {
const onlySpecAndSuites = rootSuite._getOnlyItems();
if (onlySpecAndSuites.length > 0)
return { status: 'forbid-only', locations: onlySpecAndSuites.map(specOrSuite => `${buildItemLocation(config.rootDir, specOrSuite)} > ${specOrSuite.fullTitle()}`) };
const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
if (onlyTestsAndSuites.length > 0)
return { status: 'forbid-only', locations: onlyTestsAndSuites.map(testOrSuite => `${buildItemLocation(config.rootDir, testOrSuite)} > ${testOrSuite.fullTitle()}`) };
}
const uniqueSpecs = getUniqueSpecsPerSuite(rootSuite);
if (uniqueSpecs.size > 0)
return { status: 'clashing-spec-titles', clashingSpecs: uniqueSpecs };
filterOnly(rootSuite);
filterByFocusedLine(rootSuite, testFileReFilters);
const clashingTests = getClashingTestsPerSuite(preprocessRoot);
if (clashingTests.size > 0)
return { status: 'clashing-test-titles', clashingTests: clashingTests };
filterOnly(preprocessRoot);
filterByFocusedLine(preprocessRoot, testFileReFilters);
const fileSuites = new Map<string, Suite>();
for (const fileSuite of rootSuite.suites)
for (const fileSuite of preprocessRoot.suites)
fileSuites.set(fileSuite._requireFile, fileSuite);
const outputDirs = new Set<string>();
const grepMatcher = createMatcher(config.grep);
const grepInvertMatcher = config.grepInvert ? createMatcher(config.grepInvert) : null;
const rootSuite = new Suite('');
for (const project of projects) {
for (const file of files.get(project)!) {
const fileSuite = fileSuites.get(file);
if (!fileSuite)
continue;
for (const spec of fileSuite._allSpecs()) {
const fullTitle = spec._testFullTitle(project.config.name);
if (grepInvertMatcher?.(fullTitle))
continue;
if (grepMatcher(fullTitle))
project.generateTests(spec);
for (let repeatEachIndex = 0; repeatEachIndex < project.config.repeatEach; repeatEachIndex++) {
const cloned = project.cloneSuite(fileSuite, repeatEachIndex, test => {
const fullTitle = test.fullTitle();
if (grepInvertMatcher?.(fullTitle))
return false;
return grepMatcher(fullTitle);
});
if (cloned)
rootSuite._addSuite(cloned);
}
}
outputDirs.add(project.config.outputDir);
@ -235,7 +239,7 @@ export class Runner {
if (process.stdout.isTTY) {
const workers = new Set();
rootSuite.findTest(test => {
workers.add(test.spec._requireFile + test._workerHash);
workers.add(test._requireFile + test._workerHash);
});
console.log();
const jobs = Math.min(config.workers, workers.size);
@ -259,7 +263,7 @@ export class Runner {
return { status: 'sigint' };
}
const failed = hasWorkerErrors || rootSuite.findSpec(spec => !spec.ok());
const failed = hasWorkerErrors || rootSuite.findTest(test => !test.ok());
await this._reporter.onEnd({ status: failed ? 'failed' : 'passed' });
return { status: failed ? 'failed' : 'passed' };
} finally {
@ -274,27 +278,27 @@ export class Runner {
function filterOnly(suite: Suite) {
const suiteFilter = (suite: Suite) => suite._only;
const specFilter = (spec: Spec) => spec._only;
return filterSuite(suite, suiteFilter, specFilter);
const testFilter = (test: Test) => test._only;
return filterSuite(suite, suiteFilter, testFilter);
}
function filterByFocusedLine(suite: Suite, focusedTestFileLines: FilePatternFilter[]) {
const testFileLineMatches = (specFileName: string, specLine: number) => focusedTestFileLines.some(({re, line}) => {
const testFileLineMatches = (testFileName: string, testLine: number) => focusedTestFileLines.some(({re, line}) => {
re.lastIndex = 0;
return re.test(specFileName) && (line === specLine || line === null);
return re.test(testFileName) && (line === testLine || line === null);
});
const suiteFilter = (suite: Suite) => testFileLineMatches(suite.file, suite.line);
const specFilter = (spec: Spec) => testFileLineMatches(spec.file, spec.line);
return filterSuite(suite, suiteFilter, specFilter);
const testFilter = (test: Test) => testFileLineMatches(test.file, test.line);
return filterSuite(suite, suiteFilter, testFilter);
}
function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, specFilter: (spec: Spec) => boolean) {
const onlySuites = suite.suites.filter(child => filterSuite(child, suiteFilter, specFilter) || suiteFilter(child));
const onlyTests = suite.specs.filter(specFilter);
function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: Test) => boolean) {
const onlySuites = suite.suites.filter(child => filterSuite(child, suiteFilter, testFilter) || suiteFilter(child));
const onlyTests = suite.tests.filter(testFilter);
const onlyEntries = new Set([...onlySuites, ...onlyTests]);
if (onlyEntries.size) {
suite.suites = onlySuites;
suite.specs = onlyTests;
suite.tests = onlyTests;
suite._entries = suite._entries.filter(e => onlyEntries.has(e)); // Preserve the order.
return true;
}
@ -370,29 +374,29 @@ async function collectFiles(testDir: string): Promise<string[]> {
return files;
}
function getUniqueSpecsPerSuite(rootSuite: Suite): Map<string, Spec[]> {
function visit(suite: Suite, clashingSpecs: Map<string, Spec[]>) {
function getClashingTestsPerSuite(rootSuite: Suite): Map<string, Test[]> {
function visit(suite: Suite, clashingTests: Map<string, Test[]>) {
for (const childSuite of suite.suites)
visit(childSuite, clashingSpecs);
for (const spec of suite.specs) {
const fullTitle = spec.fullTitle();
if (!clashingSpecs.has(fullTitle))
clashingSpecs.set(fullTitle, []);
clashingSpecs.set(fullTitle, clashingSpecs.get(fullTitle)!.concat(spec));
visit(childSuite, clashingTests);
for (const test of suite.tests) {
const fullTitle = test.fullTitle();
if (!clashingTests.has(fullTitle))
clashingTests.set(fullTitle, []);
clashingTests.set(fullTitle, clashingTests.get(fullTitle)!.concat(test));
}
}
const out = new Map<string, Spec[]>();
const out = new Map<string, Test[]>();
for (const fileSuite of rootSuite.suites) {
const clashingSpecs = new Map<string, Spec[]>();
visit(fileSuite, clashingSpecs);
for (const [title, specs] of clashingSpecs.entries()) {
if (specs.length > 1)
out.set(title, specs);
const clashingTests = new Map<string, Test[]>();
visit(fileSuite, clashingTests);
for (const [title, tests] of clashingTests.entries()) {
if (tests.length > 1)
out.set(title, tests);
}
}
return out;
}
function buildItemLocation(rootDir: string, specOrSuite: Suite | Spec) {
return `${path.relative(rootDir, specOrSuite.file)}:${specOrSuite.line}`;
function buildItemLocation(rootDir: string, testOrSuite: Suite | Test) {
return `${path.relative(rootDir, testOrSuite.file)}:${testOrSuite.line}`;
}

138
src/test/test.ts
View file

@ -14,6 +14,7 @@
* limitations under the License.
*/
import type { FixturePool } from './fixtures';
import * as reporterTypes from './reporter';
import type { TestTypeImpl } from './testType';
import { Annotations, Location } from './types';
@ -25,6 +26,7 @@ class Base {
column: number = 0;
parent?: Suite;
_fullTitle: string = '';
_only = false;
_requireFile: string = '';
@ -32,39 +34,15 @@ class Base {
this.title = title;
}
titlePath(): string[] {
if (!this.parent)
return [];
if (!this.title)
return this.parent.titlePath();
return [...this.parent.titlePath(), this.title];
_buildFullTitle(parentFullTitle: string) {
if (this.title)
this._fullTitle = (parentFullTitle ? parentFullTitle + ' ' : '') + this.title;
else
this._fullTitle = parentFullTitle;
}
fullTitle(): string {
return this.titlePath().join(' ');
}
}
export class Spec extends Base implements reporterTypes.Spec {
suite!: Suite;
fn: Function;
tests: Test[] = [];
_ordinalInFile: number;
_testType: TestTypeImpl;
constructor(title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl) {
super(title);
this.fn = fn;
this._ordinalInFile = ordinalInFile;
this._testType = testType;
}
ok(): boolean {
return !this.tests.find(r => !r.ok());
}
_testFullTitle(projectName: string) {
return (projectName ? `[${projectName}] ` : '') + this.fullTitle();
return this._fullTitle;
}
}
@ -77,9 +55,9 @@ export type Modifier = {
export class Suite extends Base implements reporterTypes.Suite {
suites: Suite[] = [];
specs: Spec[] = [];
tests: Test[] = [];
_fixtureOverrides: any = {};
_entries: (Suite | Spec)[] = [];
_entries: (Suite | Test)[] = [];
_hooks: {
type: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll',
fn: Function,
@ -88,12 +66,14 @@ export class Suite extends Base implements reporterTypes.Suite {
_timeout: number | undefined;
_annotations: Annotations = [];
_modifiers: Modifier[] = [];
_repeatEachIndex = 0;
_projectIndex = 0;
_addSpec(spec: Spec) {
spec.parent = this;
spec.suite = this;
this.specs.push(spec);
this._entries.push(spec);
_addTest(test: Test) {
test.parent = this;
test.suite = this;
this.tests.push(test);
this._entries.push(test);
}
_addSuite(suite: Suite) {
@ -107,21 +87,6 @@ export class Suite extends Base implements reporterTypes.Suite {
if (entry instanceof Suite) {
if (entry.findTest(fn))
return true;
} else {
for (const test of entry.tests) {
if (fn(test))
return true;
}
}
}
return false;
}
findSpec(fn: (spec: Spec) => boolean | void): boolean {
for (const entry of this._entries) {
if (entry instanceof Suite) {
if (entry.findSpec(fn))
return true;
} else {
if (fn(entry))
return true;
@ -130,48 +95,53 @@ export class Suite extends Base implements reporterTypes.Suite {
return false;
}
findSuite(fn: (suite: Suite) => boolean | void): boolean {
if (fn(this))
return true;
for (const suite of this.suites) {
if (suite.findSuite(fn))
return true;
}
return false;
}
totalTestCount(): number {
let total = 0;
for (const suite of this.suites)
total += suite.totalTestCount();
for (const spec of this.specs)
total += spec.tests.length;
total += this.tests.length;
return total;
}
_allSpecs(): Spec[] {
const result: Spec[] = [];
this.findSpec(test => { result.push(test); });
_allTests(): Test[] {
const result: Test[] = [];
this.findTest(test => { result.push(test); });
return result;
}
_getOnlyItems(): (Spec | Suite)[] {
const items: (Spec | Suite)[] = [];
_getOnlyItems(): (Test | Suite)[] {
const items: (Test | Suite)[] = [];
if (this._only)
items.push(this);
for (const suite of this.suites)
items.push(...suite._getOnlyItems());
items.push(...this.specs.filter(spec => spec._only));
items.push(...this.tests.filter(test => test._only));
return items;
}
_buildFixtureOverrides(): any {
return this.parent ? { ...this.parent._buildFixtureOverrides(), ...this._fixtureOverrides } : this._fixtureOverrides;
}
_clone(): Suite {
const suite = new Suite(this.title);
suite._only = this._only;
suite.file = this.file;
suite.line = this.line;
suite.column = this.column;
suite._requireFile = this._requireFile;
suite._fixtureOverrides = this._fixtureOverrides;
suite._hooks = this._hooks.slice();
suite._timeout = this._timeout;
suite._annotations = this._annotations.slice();
suite._modifiers = this._modifiers.slice();
return suite;
}
}
export class Test implements reporterTypes.Test {
spec: Spec;
export class Test extends Base implements reporterTypes.Test {
suite!: Suite;
fn: Function;
results: reporterTypes.TestResult[] = [];
skipped = false;
@ -181,13 +151,17 @@ export class Test implements reporterTypes.Test {
projectName = '';
retries = 0;
_ordinalInFile: number;
_testType: TestTypeImpl;
_id = '';
_repeatEachIndex = 0;
_projectIndex = 0;
_workerHash = '';
_pool: FixturePool | undefined;
constructor(spec: Spec) {
this.spec = spec;
constructor(title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl) {
super(title);
this.fn = fn;
this._ordinalInFile = ordinalInFile;
this._testType = testType;
}
status(): 'skipped' | 'expected' | 'unexpected' | 'flaky' {
@ -216,8 +190,18 @@ export class Test implements reporterTypes.Test {
return status === 'expected' || status === 'flaky' || status === 'skipped';
}
_clone(): Test {
const test = new Test(this.title, this.fn, this._ordinalInFile, this._testType);
test._only = this._only;
test.file = this.file;
test.line = this.line;
test.column = this.column;
test._requireFile = this._requireFile;
return test;
}
fullTitle(): string {
return this.spec._testFullTitle(this.projectName);
return (this.projectName ? `[${this.projectName}] ` : '') + this._fullTitle;
}
_appendTestResult(): reporterTypes.TestResult {

24
src/test/testType.ts
View file

@ -16,7 +16,7 @@
import { expect } from './expect';
import { currentlyLoadingFileSuite, currentTestInfo, setCurrentlyLoadingFileSuite } from './globals';
import { Spec, Suite } from './test';
import { Test, Suite } from './test';
import { wrapFunctionWithLocation } from './transform';
import { Fixtures, FixturesWithLocation, Location, TestType } from './types';
@ -34,9 +34,9 @@ export class TestTypeImpl {
constructor(fixtures: (FixturesWithLocation | DeclaredFixtures)[]) {
this.fixtures = fixtures;
const test: any = wrapFunctionWithLocation(this._spec.bind(this, 'default'));
const test: any = wrapFunctionWithLocation(this._createTest.bind(this, 'default'));
test.expect = expect;
test.only = wrapFunctionWithLocation(this._spec.bind(this, 'only'));
test.only = wrapFunctionWithLocation(this._createTest.bind(this, 'only'));
test.describe = wrapFunctionWithLocation(this._describe.bind(this, 'default'));
test.describe.only = wrapFunctionWithLocation(this._describe.bind(this, 'only'));
test.beforeEach = wrapFunctionWithLocation(this._hook.bind(this, 'beforeEach'));
@ -54,7 +54,7 @@ export class TestTypeImpl {
this.test = test;
}
private _spec(type: 'default' | 'only', location: Location, title: string, fn: Function) {
private _createTest(type: 'default' | 'only', location: Location, title: string, fn: Function) {
const suite = currentlyLoadingFileSuite();
if (!suite)
throw new Error(`test() can only be called in a test file`);
@ -62,15 +62,16 @@ export class TestTypeImpl {
const ordinalInFile = countByFile.get(suite._requireFile) || 0;
countByFile.set(suite._requireFile, ordinalInFile + 1);
const spec = new Spec(title, fn, ordinalInFile, this);
spec._requireFile = suite._requireFile;
spec.file = location.file;
spec.line = location.line;
spec.column = location.column;
suite._addSpec(spec);
const test = new Test(title, fn, ordinalInFile, this);
test._requireFile = suite._requireFile;
test.file = location.file;
test.line = location.line;
test.column = location.column;
suite._addTest(test);
test._buildFullTitle(suite.fullTitle());
if (type === 'only')
spec._only = true;
test._only = true;
}
private _describe(type: 'default' | 'only', location: Location, title: string, fn: Function) {
@ -84,6 +85,7 @@ export class TestTypeImpl {
child.line = location.line;
child.column = location.column;
suite._addSuite(child);
child._buildFullTitle(suite.fullTitle());
if (type === 'only')
child._only = true;

66
src/test/workerRunner.ts
View file

@ -23,10 +23,10 @@ import { monotonicTime, DeadlineRunner, raceAgainstDeadline, serializeError } fr
import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams } from './ipc';
import { setCurrentTestInfo } from './globals';
import { Loader } from './loader';
import { Modifier, Spec, Suite, Test } from './test';
import { Modifier, Suite, Test } from './test';
import { Annotations, TestError, TestInfo, WorkerInfo } from './types';
import { ProjectImpl } from './project';
import { FixtureRunner } from './fixtures';
import { FixturePool, FixtureRunner } from './fixtures';
const removeFolderAsync = util.promisify(rimraf);
@ -116,20 +116,22 @@ export class WorkerRunner extends EventEmitter {
this._entries = new Map(runPayload.entries.map(e => [ e.testId, e ]));
await this._loadIfNeeded();
const fileSuite = await this._loader.loadTestFile(runPayload.file);
let anySpec: Spec | undefined;
fileSuite.findSpec(spec => {
const test = this._project.generateTests(spec, this._params.repeatEachIndex)[0];
if (this._entries.has(test._id))
anySpec = spec;
let anyPool: FixturePool | undefined;
const suite = this._project.cloneSuite(fileSuite, this._params.repeatEachIndex, test => {
if (!this._entries.has(test._id))
return false;
anyPool = test._pool;
return true;
});
if (!anySpec) {
if (!suite || !anyPool) {
this._reportDone();
return;
}
this._fixtureRunner.setPool(this._project.buildPool(anySpec));
await this._runSuite(fileSuite, []);
this._fixtureRunner.setPool(anyPool);
await this._runSuite(suite, []);
if (this._isStopped)
return;
@ -156,7 +158,7 @@ export class WorkerRunner extends EventEmitter {
annotations.push({ type: beforeAllModifier.type, description: beforeAllModifier.description });
}
const skipHooks = !this._hasTestsToRun(suite) || annotations.some(a => a.type === 'fixme' || a.type === 'skip');
const skipHooks = annotations.some(a => a.type === 'fixme' || a.type === 'skip');
for (const hook of suite._hooks) {
if (hook.type !== 'beforeAll' || skipHooks)
continue;
@ -173,7 +175,7 @@ export class WorkerRunner extends EventEmitter {
if (entry instanceof Suite)
await this._runSuite(entry, annotations);
else
await this._runSpec(entry, annotations);
await this._runTest(entry, annotations);
}
for (const hook of suite._hooks) {
if (hook.type !== 'afterAll' || skipHooks)
@ -189,10 +191,9 @@ export class WorkerRunner extends EventEmitter {
}
}
private async _runSpec(spec: Spec, annotations: Annotations) {
private async _runTest(test: Test, annotations: Annotations) {
if (this._isStopped)
return;
const test = spec.tests[0];
const entry = this._entries.get(test._id);
if (!entry)
return;
@ -202,9 +203,9 @@ export class WorkerRunner extends EventEmitter {
const testId = test._id;
const baseOutputDir = (() => {
const relativeTestFilePath = path.relative(this._project.config.testDir, spec._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, ''));
const relativeTestFilePath = path.relative(this._project.config.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, ''));
const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-');
let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(spec.title);
let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(test.title);
if (this._uniqueProjectNamePathSegment)
testOutputDir += '-' + this._uniqueProjectNamePathSegment;
if (entry.retry)
@ -216,11 +217,11 @@ export class WorkerRunner extends EventEmitter {
const testInfo: TestInfo = {
...this._workerInfo,
title: spec.title,
file: spec.file,
line: spec.line,
column: spec.column,
fn: spec.fn,
title: test.title,
file: test.file,
line: test.line,
column: test.column,
fn: test.fn,
repeatEachIndex: this._params.repeatEachIndex,
retry: entry.retry,
expectedStatus: 'passed',
@ -248,7 +249,7 @@ export class WorkerRunner extends EventEmitter {
snapshotName = sanitizeForFilePath(snapshotName.substring(0, snapshotName.length - ext.length)) + suffix + ext;
else
snapshotName = sanitizeForFilePath(snapshotName) + suffix;
return path.join(spec._requireFile + '-snapshots', snapshotName);
return path.join(test._requireFile + '-snapshots', snapshotName);
},
skip: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'skip', args),
fixme: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fixme', args),
@ -262,7 +263,7 @@ export class WorkerRunner extends EventEmitter {
};
// Inherit test.setTimeout() from parent suites.
for (let suite = spec.parent; suite; suite = suite.parent) {
for (let suite = test.parent; suite; suite = suite.parent) {
if (suite._timeout !== undefined) {
testInfo.setTimeout(suite._timeout);
break;
@ -301,7 +302,7 @@ export class WorkerRunner extends EventEmitter {
}
// Update the fixture pool - it may differ between tests, but only in test-scoped fixtures.
this._fixtureRunner.setPool(this._project.buildPool(spec));
this._fixtureRunner.setPool(test._pool!);
deadlineRunner = new DeadlineRunner(this._runTestWithBeforeHooks(test, testInfo), deadline());
const result = await deadlineRunner.result;
@ -352,7 +353,7 @@ export class WorkerRunner extends EventEmitter {
private async _runTestWithBeforeHooks(test: Test, testInfo: TestInfo) {
try {
const beforeEachModifiers: Modifier[] = [];
for (let s = test.spec.parent; s; s = s.parent) {
for (let s = test.parent; s; s = s.parent) {
const modifiers = s._modifiers.filter(modifier => !this._fixtureRunner.dependsOnWorkerFixturesOnly(modifier.fn, modifier.location));
beforeEachModifiers.push(...modifiers.reverse());
}
@ -363,7 +364,7 @@ export class WorkerRunner extends EventEmitter {
const result = await this._fixtureRunner.resolveParametersAndRunHookOrTest(modifier.fn, 'test', testInfo);
testInfo[modifier.type](!!result, modifier.description!);
}
await this._runHooks(test.spec.parent!, 'beforeEach', testInfo);
await this._runHooks(test.parent!, 'beforeEach', testInfo);
} catch (error) {
if (error instanceof SkipError) {
if (testInfo.status === 'passed')
@ -380,7 +381,7 @@ export class WorkerRunner extends EventEmitter {
return;
try {
await this._fixtureRunner.resolveParametersAndRunHookOrTest(test.spec.fn, 'test', testInfo);
await this._fixtureRunner.resolveParametersAndRunHookOrTest(test.fn, 'test', testInfo);
} catch (error) {
if (error instanceof SkipError) {
if (testInfo.status === 'passed')
@ -399,7 +400,7 @@ export class WorkerRunner extends EventEmitter {
private async _runAfterHooks(test: Test, testInfo: TestInfo) {
try {
await this._runHooks(test.spec.parent!, 'afterEach', testInfo);
await this._runHooks(test.parent!, 'afterEach', testInfo);
} catch (error) {
if (!(error instanceof SkipError)) {
if (testInfo.status === 'passed')
@ -458,13 +459,6 @@ export class WorkerRunner extends EventEmitter {
this._reportDone();
this.stop();
}
private _hasTestsToRun(suite: Suite): boolean {
return suite.findSpec(spec => {
const entry = this._entries.get(spec.tests[0]._id);
return !!entry;
});
}
}
function buildTestBeginPayload(testId: string, testInfo: TestInfo): TestBeginPayload {

62
tests/playwright-test/config.spec.ts
View file

@ -321,65 +321,3 @@ test('should work with undefined values and base', async ({ runInlineTest }) =>
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
});
test('should work with custom reporter', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': `
class Reporter {
constructor(options) {
this.options = options;
}
onBegin() {
console.log('\\n%%reporter-begin%%' + this.options.begin);
}
onTestBegin() {
console.log('\\n%%reporter-testbegin%%');
}
onStdOut() {
console.log('\\n%%reporter-stdout%%');
}
onStdErr() {
console.log('\\n%%reporter-stderr%%');
}
onTestEnd() {
console.log('\\n%%reporter-testend%%');
}
onTimeout() {
console.log('\\n%%reporter-timeout%%');
}
onError() {
console.log('\\n%%reporter-error%%');
}
async onEnd() {
await new Promise(f => setTimeout(f, 500));
console.log('\\n%%reporter-end%%' + this.options.end);
}
}
export default Reporter;
`,
'playwright.config.ts': `
module.exports = {
reporter: [
[ './reporter.ts', { begin: 'begin', end: 'end' } ]
]
};
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async ({}) => {
console.log('log');
console.error('error');
});
`
}, { reporter: '' });
expect(result.exitCode).toBe(0);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%reporter-begin%%begin',
'%%reporter-testbegin%%',
'%%reporter-stdout%%',
'%%reporter-stderr%%',
'%%reporter-testend%%',
'%%reporter-end%%end',
]);
});

24
tests/playwright-test/hooks.spec.ts
View file

@ -190,3 +190,27 @@ test('beforeAll from a helper file should throw', async ({ runInlineTest }) => {
expect(result.exitCode).toBe(1);
expect(result.output).toContain('beforeAll hook can only be called in a test file');
});
test('beforeAll hooks are skipped when no tests in the suite are run', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
test.describe('suite1', () => {
test.beforeAll(() => {
console.log('\\n%%beforeAll1');
});
test('skipped', () => {});
});
test.describe('suite2', () => {
test.beforeAll(() => {
console.log('\\n%%beforeAll2');
});
test.only('passed', () => {});
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.output).toContain('%%beforeAll2');
expect(result.output).not.toContain('%%beforeAll1');
});

18
tests/playwright-test/junit-reporter.spec.ts
View file

@ -205,17 +205,23 @@ test('should render projects', async ({ runInlineTest }) => {
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('2');
expect(xml['testsuites']['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'].length).toBe(1);
expect(xml['testsuites']['testsuite'].length).toBe(2);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('[project1] one');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toContain('[project1] one');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toContain('a.test.js:6:7');
expect(xml['testsuites']['testsuite'][0]['testcase'][1]['$']['name']).toBe('one');
expect(xml['testsuites']['testsuite'][0]['testcase'][1]['$']['classname']).toContain('[project2] one');
expect(xml['testsuites']['testsuite'][0]['testcase'][1]['$']['classname']).toContain('a.test.js:6:7');
expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][1]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][1]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][1]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['name']).toBe('[project2] one');
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toContain('[project2] one');
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toContain('a.test.js:6:7');
expect(result.exitCode).toBe(0);
});

3
tests/playwright-test/match-grep.spec.ts
View file

@ -64,12 +64,14 @@ const files = {
test('should grep test name', async ({ runInlineTest }) => {
const result = await runInlineTest(files, { 'grep': 'test [A-B]' });
expect(result.passed).toBe(6);
expect(result.skipped).toBe(0);
expect(result.exitCode).toBe(0);
});
test('should grep test name with regular expression', async ({ runInlineTest }) => {
const result = await runInlineTest(files, { 'grep': '/B$/' });
expect(result.passed).toBe(3);
expect(result.skipped).toBe(0);
expect(result.exitCode).toBe(0);
});
@ -100,5 +102,6 @@ test('should grep by project name', async ({ runInlineTest }) => {
test('should grep invert test name', async ({ runInlineTest }) => {
const result = await runInlineTest(files, { 'grep-invert': 'BB' });
expect(result.passed).toBe(6);
expect(result.skipped).toBe(0);
expect(result.exitCode).toBe(0);
});

91
tests/playwright-test/reporter.spec.ts Normal file
View file

@ -0,0 +1,91 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { test, expect } from './playwright-test-fixtures';
test('should work with custom reporter', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': `
class Reporter {
constructor(options) {
this.options = options;
}
onBegin(config, suite) {
console.log('\\n%%reporter-begin-' + this.options.begin + '-' + suite.suites.length + '%%');
}
onTestBegin(test) {
console.log('\\n%%reporter-testbegin-' + test.title + '-' + test.projectName + '%%');
}
onStdOut() {
console.log('\\n%%reporter-stdout%%');
}
onStdErr() {
console.log('\\n%%reporter-stderr%%');
}
onTestEnd(test) {
console.log('\\n%%reporter-testend-' + test.title + '-' + test.projectName + '%%');
}
onTimeout() {
console.log('\\n%%reporter-timeout%%');
}
onError() {
console.log('\\n%%reporter-error%%');
}
async onEnd() {
await new Promise(f => setTimeout(f, 500));
console.log('\\n%%reporter-end-' + this.options.end + '%%');
}
}
export default Reporter;
`,
'playwright.config.ts': `
module.exports = {
reporter: [
[ './reporter.ts', { begin: 'begin', end: 'end' } ]
],
projects: [
{ name: 'foo', repeatEach: 2 },
{ name: 'bar' },
],
};
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async ({}) => {
console.log('log');
console.error('error');
});
`
}, { reporter: '', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%reporter-begin-begin-3%%',
'%%reporter-testbegin-pass-foo%%',
'%%reporter-stdout%%',
'%%reporter-stderr%%',
'%%reporter-testend-pass-foo%%',
'%%reporter-testbegin-pass-foo%%',
'%%reporter-stdout%%',
'%%reporter-stderr%%',
'%%reporter-testend-pass-foo%%',
'%%reporter-testbegin-pass-bar%%',
'%%reporter-stdout%%',
'%%reporter-stderr%%',
'%%reporter-testend-pass-bar%%',
'%%reporter-end-end%%',
]);
});