Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore(test-runner): revert recent changes to fix tests (#12439)

Browse source 
* Revert "fix(hooks): separate test timeout from beforeAll/afterAll timeouts (#12413)"

This reverts commit 73dee69558.

* Revert "fix(test-runner): rely on test title paths instead of ordinal (#12414)"

This reverts commit d744a87aee.

* Revert "chore(test runner): run hooks/modifiers as a part of the test  (#12329)"

This reverts commit 47045ba48d.
This commit is contained in:
Pavel Feldman 2022-03-01 09:11:17 -08:00 committed by GitHub
parent d2ae6a9db2
commit 6a663ef54f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 389 additions and 659 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
packages/playwright-core/src/utils/async.ts
View file

@ -19,7 +19,7 @@ import { monotonicTime } from './utils';
export class TimeoutRunnerError extends Error {}
type TimeoutRunnerData = {
lastElapsedSync: number,
start: number,
timer: NodeJS.Timer | undefined,
timeoutPromise: ManualPromise<any>,
};
@ -35,7 +35,7 @@ export class TimeoutRunner {
async run<T>(cb: () => Promise<T>): Promise<T> {
const running = this._running = {
lastElapsedSync: monotonicTime(),
start: monotonicTime(),
timer: undefined,
timeoutPromise: new ManualPromise(),
};
@ -47,6 +47,7 @@ export class TimeoutRunner {
this._updateTimeout(running, this._timeout);
return await resultPromise;
} finally {
this._elapsed += monotonicTime() - running.start;
this._updateTimeout(running, 0);
if (this._running === running)
this._running = undefined;
@ -58,27 +59,15 @@ export class TimeoutRunner {
this._updateTimeout(this._running, -1);
}
elapsed() {
this._syncElapsedAndStart();
return this._elapsed;
}
updateTimeout(timeout: number, elapsed?: number) {
updateTimeout(timeout: number) {
this._timeout = timeout;
if (elapsed !== undefined) {
this._syncElapsedAndStart();
this._elapsed = elapsed;
}
if (this._running)
this._updateTimeout(this._running, timeout);
}
private _syncElapsedAndStart() {
if (this._running) {
const now = monotonicTime();
this._elapsed += now - this._running.lastElapsedSync;
this._running.lastElapsedSync = now;
}
resetTimeout(timeout: number) {
this._elapsed = 0;
this.updateTimeout(timeout);
}
private _updateTimeout(running: TimeoutRunnerData, timeout: number) {
@ -86,10 +75,10 @@ export class TimeoutRunner {
clearTimeout(running.timer);
running.timer = undefined;
}
this._syncElapsedAndStart();
if (timeout === 0)
return;
timeout = timeout - this._elapsed;
const elapsed = (monotonicTime() - running.start) + this._elapsed;
timeout = timeout - elapsed;
if (timeout <= 0)
running.timeoutPromise.reject(new TimeoutRunnerError());
else

57
packages/playwright-test/src/dispatcher.ts
View file

@ -59,8 +59,15 @@ export class Dispatcher {
this._queue = testGroups;
for (const group of testGroups) {
this._queueHashCount.set(group.workerHash, 1 + (this._queueHashCount.get(group.workerHash) || 0));
for (const test of group.tests)
for (const test of group.tests) {
this._testById.set(test._id, { test, resultByWorkerIndex: new Map() });
for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) {
for (const hook of suite.hooks) {
if (!this._testById.has(hook._id))
this._testById.set(hook._id, { test: hook, resultByWorkerIndex: new Map() });
}
}
}
}
}
@ -177,20 +184,25 @@ export class Dispatcher {
const remainingByTestId = new Map(testGroup.tests.map(e => [ e._id, e ]));
const failedTestIds = new Set<string>();
let runningHookId: string | undefined;
const onTestBegin = (params: TestBeginPayload) => {
const data = this._testById.get(params.testId)!;
if (data.test._type !== 'test')
runningHookId = params.testId;
if (this._hasReachedMaxFailures())
return;
const result = data.test._appendTestResult();
data.resultByWorkerIndex.set(worker.workerIndex, { result, stepStack: new Set(), steps: new Map() });
result.workerIndex = worker.workerIndex;
result.startTime = new Date(params.startWallTime);
this._reporter.onTestBegin?.(data.test, result);
if (data.test._type === 'test')
this._reporter.onTestBegin?.(data.test, result);
};
worker.addListener('testBegin', onTestBegin);
const onTestEnd = (params: TestEndPayload) => {
runningHookId = undefined;
remainingByTestId.delete(params.testId);
if (this._hasReachedMaxFailures())
return;
@ -212,7 +224,7 @@ export class Dispatcher {
test.annotations = params.annotations;
test.timeout = params.timeout;
const isFailure = result.status !== 'skipped' && result.status !== test.expectedStatus;
if (isFailure)
if (isFailure && test._type === 'test')
failedTestIds.add(params.testId);
this._reportTestEnd(test, result);
};
@ -278,7 +290,7 @@ export class Dispatcher {
// - there are no remaining
// - we are here not because something failed
// - no unrecoverable worker error
if (!remaining.length && !failedTestIds.size && !params.fatalErrors.length && !params.skipRemaining) {
if (!remaining.length && !failedTestIds.size && !params.fatalErrors.length) {
if (this._isWorkerRedundant(worker))
worker.stop();
doneWithJob();
@ -290,8 +302,18 @@ export class Dispatcher {
// In case of fatal error, report first remaining test as failing with this error,
// and all others as skipped.
if (params.fatalErrors.length || params.skipRemaining) {
let shouldAddFatalErrorsToNextTest = params.fatalErrors.length > 0;
if (params.fatalErrors.length) {
// Perhaps we were running a hook - report it as failed.
if (runningHookId) {
const data = this._testById.get(runningHookId)!;
const { result } = data.resultByWorkerIndex.get(worker.workerIndex)!;
result.errors = [...params.fatalErrors];
result.error = result.errors[0];
result.status = 'failed';
this._reporter.onTestEnd?.(data.test, result);
}
let first = true;
for (const test of remaining) {
if (this._hasReachedMaxFailures())
break;
@ -303,23 +325,24 @@ export class Dispatcher {
result = runData.result;
} else {
result = data.test._appendTestResult();
this._reporter.onTestBegin?.(test, result);
if (test._type === 'test')
this._reporter.onTestBegin?.(test, result);
}
result.errors = shouldAddFatalErrorsToNextTest ? [...params.fatalErrors] : [];
result.errors = [...params.fatalErrors];
result.error = result.errors[0];
result.status = shouldAddFatalErrorsToNextTest ? 'failed' : 'skipped';
result.status = first ? 'failed' : 'skipped';
this._reportTestEnd(test, result);
failedTestIds.add(test._id);
shouldAddFatalErrorsToNextTest = false;
first = false;
}
if (shouldAddFatalErrorsToNextTest) {
if (first) {
// We had a fatal error after all tests have passed - most likely in the afterAll hook.
// Let's just fail the test run.
this._hasWorkerErrors = true;
for (const error of params.fatalErrors)
this._reporter.onError?.(error);
}
// Since we pretend that all remaining tests failed/skipped, there is nothing else to run,
// Since we pretend that all remaining tests failed, there is nothing else to run,
// except for possible retries.
remaining = [];
}
@ -352,7 +375,8 @@ export class Dispatcher {
// Emulate a "skipped" run, and drop this test from remaining.
const result = test._appendTestResult();
this._reporter.onTestBegin?.(test, result);
if (test._type === 'test')
this._reporter.onTestBegin?.(test, result);
result.status = 'skipped';
this._reportTestEnd(test, result);
return false;
@ -384,7 +408,7 @@ export class Dispatcher {
worker.on('done', onDone);
const onExit = (expectedly: boolean) => {
onDone({ skipRemaining: false, fatalErrors: expectedly ? [] : [{ value: 'Worker process exited unexpectedly' }] });
onDone({ fatalErrors: expectedly ? [] : [{ value: 'Worker process exited unexpectedly' }] });
};
worker.on('exit', onExit);
@ -436,9 +460,10 @@ export class Dispatcher {
}
private _reportTestEnd(test: TestCase, result: TestResult) {
if (result.status !== 'skipped' && result.status !== test.expectedStatus)
if (test._type === 'test' && result.status !== 'skipped' && result.status !== test.expectedStatus)
++this._failureCount;
this._reporter.onTestEnd?.(test, result);
if (test._type === 'test')
this._reporter.onTestEnd?.(test, result);
const maxFailures = this._loader.fullConfig().maxFailures;
if (maxFailures && this._failureCount === maxFailures)
this.stop().catch(e => {});

21
packages/playwright-test/src/fixtures.ts
View file

@ -51,7 +51,7 @@ class Fixture {
this.value = null;
}
async setup(testInfo: TestInfo) {
async setup(workerInfo: WorkerInfo, testInfo: TestInfo | undefined) {
if (typeof this.registration.fn !== 'function') {
this.value = this.registration.fn;
return;
@ -60,7 +60,7 @@ class Fixture {
const params: { [key: string]: any } = {};
for (const name of this.registration.deps) {
const registration = this.runner.pool!.resolveDependency(this.registration, name)!;
const dep = await this.runner.setupFixtureForRegistration(registration, testInfo);
const dep = await this.runner.setupFixtureForRegistration(registration, workerInfo, testInfo);
dep.usages.add(this);
params[name] = dep.value;
}
@ -77,7 +77,6 @@ class Fixture {
useFuncStarted.resolve();
await this._useFuncFinished;
};
const workerInfo: WorkerInfo = { config: testInfo.config, parallelIndex: testInfo.parallelIndex, workerIndex: testInfo.workerIndex, project: testInfo.project };
const info = this.registration.scope === 'worker' ? workerInfo : testInfo;
this._selfTeardownComplete = Promise.resolve().then(() => this.registration.fn(params, useFunc, info)).catch((e: any) => {
if (!useFuncStarted.isDone())
@ -262,12 +261,12 @@ export class FixtureRunner {
throw error;
}
async resolveParametersForFunction(fn: Function, testInfo: TestInfo): Promise<object> {
async resolveParametersForFunction(fn: Function, workerInfo: WorkerInfo, testInfo: TestInfo | undefined): Promise<object> {
// Install all automatic fixtures.
for (const registration of this.pool!.registrations.values()) {
const shouldSkip = !testInfo && registration.scope === 'test';
if (registration.auto && !shouldSkip)
await this.setupFixtureForRegistration(registration, testInfo);
await this.setupFixtureForRegistration(registration, workerInfo, testInfo);
}
// Install used fixtures.
@ -275,18 +274,18 @@ export class FixtureRunner {
const params: { [key: string]: any } = {};
for (const name of names) {
const registration = this.pool!.registrations.get(name)!;
const fixture = await this.setupFixtureForRegistration(registration, testInfo);
const fixture = await this.setupFixtureForRegistration(registration, workerInfo, testInfo);
params[name] = fixture.value;
}
return params;
}
async resolveParametersAndRunFunction(fn: Function, testInfo: TestInfo) {
const params = await this.resolveParametersForFunction(fn, testInfo);
return fn(params, testInfo);
async resolveParametersAndRunFunction(fn: Function, workerInfo: WorkerInfo, testInfo: TestInfo | undefined) {
const params = await this.resolveParametersForFunction(fn, workerInfo, testInfo);
return fn(params, testInfo || workerInfo);
}
async setupFixtureForRegistration(registration: FixtureRegistration, testInfo: TestInfo): Promise<Fixture> {
async setupFixtureForRegistration(registration: FixtureRegistration, workerInfo: WorkerInfo, testInfo: TestInfo | undefined): Promise<Fixture> {
if (registration.scope === 'test')
this.testScopeClean = false;
@ -296,7 +295,7 @@ export class FixtureRunner {
fixture = new Fixture(this, registration);
this.instanceForId.set(registration.id, fixture);
await fixture.setup(testInfo);
await fixture.setup(workerInfo, testInfo);
return fixture;
}

6
packages/playwright-test/src/index.ts
View file

@ -423,7 +423,7 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
}));
// 7. Cleanup created contexts when we know it's safe - this will produce nice error message.
if (testInfo.status === 'timedOut' && testInfo.errors.some(error => error.message?.match(/Timeout of \d+ms exceeded in beforeAll hook./))) {
if (hookType(testInfo) === 'beforeAll' && testInfo.status === 'timedOut') {
const anyContext = leftoverContexts[0];
const pendingCalls = anyContext ? formatPendingCalls((anyContext as any)._connection.pendingProtocolCalls()) : '';
await Promise.all(leftoverContexts.filter(c => createdContexts.has(c)).map(c => c.close()));
@ -519,9 +519,9 @@ function formatStackFrame(frame: StackFrame) {
}
function hookType(testInfo: TestInfo): 'beforeAll' | 'afterAll' | undefined {
if ((testInfo as any)._currentRunnable?.type === 'beforeAll')
if (testInfo.title.startsWith('beforeAll'))
return 'beforeAll';
if ((testInfo as any)._currentRunnable?.type === 'afterAll')
if (testInfo.title.startsWith('afterAll'))
return 'afterAll';
}

1
packages/playwright-test/src/ipc.ts
View file

@ -76,7 +76,6 @@ export type RunPayload = {
export type DonePayload = {
fatalErrors: TestError[];
skipRemaining: boolean;
};
export type TestOutputPayload = {

24
packages/playwright-test/src/project.ts
View file

@ -18,7 +18,6 @@ import type { FullProject, Fixtures, FixturesWithLocation } from './types';
import { Suite, TestCase } from './test';
import { FixturePool, isFixtureOption } from './fixtures';
import { TestTypeImpl } from './testType';
import { calculateSha1 } from 'playwright-core/lib/utils/utils';
export class ProjectImpl {
config: FullProject;
@ -53,8 +52,10 @@ export class ProjectImpl {
for (const parent of parents) {
if (parent._use.length)
pool = new FixturePool(parent._use, pool, parent._isDescribe);
for (const hook of parent._hooks)
for (const hook of parent._eachHooks)
pool.validateFunction(hook.fn, hook.type + ' hook', hook.location);
for (const hook of parent.hooks)
pool.validateFunction(hook.fn, hook._type + ' hook', hook.location);
for (const modifier of parent._modifiers)
pool.validateFunction(modifier.fn, modifier.type + ' modifier', modifier.location);
}
@ -65,21 +66,19 @@ export class ProjectImpl {
return this.testPools.get(test)!;
}
private _cloneEntries(from: Suite, to: Suite, repeatEachIndex: number, filter: (test: TestCase) => boolean, relativeTitlePath: string): boolean {
private _cloneEntries(from: Suite, to: Suite, repeatEachIndex: number, filter: (test: TestCase) => boolean): boolean {
for (const entry of from._entries) {
if (entry instanceof Suite) {
const suite = entry._clone();
to._addSuite(suite);
if (!this._cloneEntries(entry, suite, repeatEachIndex, filter, relativeTitlePath + ' ' + suite.title)) {
if (!this._cloneEntries(entry, suite, repeatEachIndex, filter)) {
to._entries.pop();
to.suites.pop();
}
} else {
const test = entry._clone();
test.retries = this.config.retries;
// We rely upon relative paths being unique.
// See `getClashingTestsPerSuite()` in `runner.ts`.
test._id = `${calculateSha1(relativeTitlePath + ' ' + entry.title)}@${entry._requireFile}#run${this.index}-repeat${repeatEachIndex}`;
test._id = `${entry._ordinalInFile}@${entry._requireFile}#run${this.index}-repeat${repeatEachIndex}`;
test.repeatEachIndex = repeatEachIndex;
test._projectIndex = this.index;
to._addTest(test);
@ -95,12 +94,21 @@ export class ProjectImpl {
}
if (!to._entries.length)
return false;
for (const hook of from.hooks) {
const clone = hook._clone();
clone.retries = 1;
clone._pool = this.buildPool(hook);
clone._projectIndex = this.index;
clone._id = `${hook._ordinalInFile}@${hook._requireFile}#run${this.index}-repeat${repeatEachIndex}`;
clone.repeatEachIndex = repeatEachIndex;
to._addAllHook(clone);
}
return true;
}
cloneFileSuite(suite: Suite, repeatEachIndex: number, filter: (test: TestCase) => boolean): Suite | undefined {
const result = suite._clone();
return this._cloneEntries(suite, result, repeatEachIndex, filter, '') ? result : undefined;
return this._cloneEntries(suite, result, repeatEachIndex, filter) ? result : undefined;
}
private resolveFixtures(testType: TestTypeImpl, configUse: Fixtures): FixturesWithLocation[] {

3
packages/playwright-test/src/reporters/line.ts
View file

@ -61,7 +61,8 @@ class LineReporter extends BaseReporter {
override onTestEnd(test: TestCase, result: TestResult) {
super.onTestEnd(test, result);
++this._current;
if (!test.title.startsWith('beforeAll') && !test.title.startsWith('afterAll'))
++this._current;
const retriesSuffix = this.totalTestCount < this._current ? ` (retries)` : ``;
const title = `[${this._current}/${this.totalTestCount}]${retriesSuffix} ${formatTestTitle(this.config, test)}`;
const suffix = result.retry ? ` (retry #${result.retry})` : '';

24
packages/playwright-test/src/test.ts
View file

@ -17,7 +17,7 @@
import type { FixturePool } from './fixtures';
import * as reporterTypes from '../types/testReporter';
import type { TestTypeImpl } from './testType';
import { Annotation, FixturesWithLocation, Location } from './types';
import { Annotations, FixturesWithLocation, Location, TestCaseType } from './types';
import { FullProject } from './types';
class Base {
@ -45,9 +45,10 @@ export class Suite extends Base implements reporterTypes.Suite {
_use: FixturesWithLocation[] = [];
_isDescribe = false;
_entries: (Suite | TestCase)[] = [];
_hooks: { type: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll', fn: Function, location: Location }[] = [];
hooks: TestCase[] = [];
_eachHooks: { type: 'beforeEach' | 'afterEach', fn: Function, location: Location }[] = [];
_timeout: number | undefined;
_annotations: Annotation[] = [];
_annotations: Annotations = [];
_modifiers: Modifier[] = [];
_parallelMode: 'default' | 'serial' | 'parallel' = 'default';
_projectConfig: FullProject | undefined;
@ -65,6 +66,11 @@ export class Suite extends Base implements reporterTypes.Suite {
this._entries.push(suite);
}
_addAllHook(hook: TestCase) {
hook.parent = this;
this.hooks.push(hook);
}
allTests(): TestCase[] {
const result: TestCase[] = [];
const visit = (suite: Suite) => {
@ -101,7 +107,7 @@ export class Suite extends Base implements reporterTypes.Suite {
suite.location = this.location;
suite._requireFile = this._requireFile;
suite._use = this._use.slice();
suite._hooks = this._hooks.slice();
suite._eachHooks = this._eachHooks.slice();
suite._timeout = this._timeout;
suite._annotations = this._annotations.slice();
suite._modifiers = this._modifiers.slice();
@ -124,19 +130,23 @@ export class TestCase extends Base implements reporterTypes.TestCase {
expectedStatus: reporterTypes.TestStatus = 'passed';
timeout = 0;
annotations: Annotation[] = [];
annotations: Annotations = [];
retries = 0;
repeatEachIndex = 0;
_type: TestCaseType;
_ordinalInFile: number;
_testType: TestTypeImpl;
_id = '';
_workerHash = '';
_pool: FixturePool | undefined;
_projectIndex = 0;
constructor(title: string, fn: Function, testType: TestTypeImpl, location: Location) {
constructor(type: TestCaseType, title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl, location: Location) {
super(title);
this._type = type;
this.fn = fn;
this._ordinalInFile = ordinalInFile;
this._testType = testType;
this.location = location;
}
@ -164,7 +174,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
}
_clone(): TestCase {
const test = new TestCase(this.title, this.fn, this._testType, this.location);
const test = new TestCase(this._type, this.title, this.fn, this._ordinalInFile, this._testType, this.location);
test._only = this._only;
test._requireFile = this._requireFile;
test.expectedStatus = this.expectedStatus;

78
packages/playwright-test/src/testInfo.ts
View file

@ -25,15 +25,8 @@ import { WorkerInitParams } from './ipc';
import { Loader } from './loader';
import { ProjectImpl } from './project';
import { TestCase } from './test';
import { Annotation, TestStepInternal, Location } from './types';
import { addSuffixToFilePath, getContainedPath, monotonicTime, sanitizeForFilePath, serializeError, trimLongString } from './util';
type RunnableDescription = {
type: 'test' | 'beforeAll' | 'afterAll' | 'beforeEach' | 'afterEach' | 'slow' | 'skip' | 'fail' | 'fixme' | 'teardown';
location?: Location;
// When runnable has a separate timeout, it does not count into the "shared time pool" for the test.
timeout?: number;
};
import { Annotations, TestStepInternal } from './types';
import { addSuffixToFilePath, formatLocation, getContainedPath, monotonicTime, sanitizeForFilePath, serializeError, trimLongString } from './util';
export class TestInfoImpl implements TestInfo {
private _projectImpl: ProjectImpl;
@ -43,9 +36,6 @@ export class TestInfoImpl implements TestInfo {
readonly _startTime: number;
readonly _startWallTime: number;
private _hasHardError: boolean = false;
private _currentRunnable: RunnableDescription = { type: 'test' };
// Holds elapsed time of the "time pool" shared between fixtures, each hooks and test itself.
private _elapsedTestTime = 0;
// ------------ TestInfo fields ------------
readonly repeatEachIndex: number;
@ -62,7 +52,7 @@ export class TestInfoImpl implements TestInfo {
readonly fn: Function;
expectedStatus: TestStatus;
duration: number = 0;
readonly annotations: Annotation[] = [];
readonly annotations: Annotations = [];
readonly attachments: TestInfo['attachments'] = [];
status: TestStatus = 'passed';
readonly stdout: TestInfo['stdout'] = [];
@ -126,7 +116,7 @@ export class TestInfoImpl implements TestInfo {
const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, ''));
const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-');
const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ');
const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ') + (test._type === 'test' ? '' : '-worker' + this.workerIndex);
let testOutputDir = trimLongString(sanitizedRelativePath + '-' + sanitizeForFilePath(fullTitleWithoutSpec));
if (uniqueProjectNamePathSegment)
@ -171,16 +161,6 @@ export class TestInfoImpl implements TestInfo {
}
}
_setCurrentRunnable(runnable: RunnableDescription) {
if (this._currentRunnable.timeout === undefined)
this._elapsedTestTime = this._timeoutRunner.elapsed();
this._currentRunnable = runnable;
if (runnable.timeout === undefined)
this._timeoutRunner.updateTimeout(this.timeout, this._elapsedTestTime);
else
this._timeoutRunner.updateTimeout(runnable.timeout, 0);
}
async _runWithTimeout(cb: () => Promise<any>): Promise<void> {
try {
await this._timeoutRunner.run(cb);
@ -190,15 +170,13 @@ export class TestInfoImpl implements TestInfo {
// Do not overwrite existing failure upon hook/teardown timeout.
if (this.status === 'passed') {
this.status = 'timedOut';
const title = titleForRunnable(this._currentRunnable);
const suffix = title ? ` in ${title}` : '';
const message = colors.red(`Timeout of ${this._currentRunnable.timeout ?? this.timeout}ms exceeded${suffix}.`);
const location = this._currentRunnable.location;
this.errors.push({
message,
// Include location for hooks and modifiers to distinguish between them.
stack: location ? message + `\n at ${location.file}:${location.line}:${location.column}` : undefined,
});
if (this._test._type === 'test') {
this.errors.push({ message: colors.red(`Timeout of ${this.timeout}ms exceeded.`) });
} else {
// Include location for the hook to distinguish between multiple hooks.
const message = colors.red(`Timeout of ${this.timeout}ms exceeded in ${this._test._type} hook.`);
this.errors.push({ message: message, stack: message + `\n at ${formatLocation(this._test.location)}.` });
}
}
}
this.duration = monotonicTime() - this._startTime;
@ -295,38 +273,12 @@ export class TestInfoImpl implements TestInfo {
}
setTimeout(timeout: number) {
if (this._currentRunnable.timeout !== undefined) {
if (!this._currentRunnable.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
this._currentRunnable.timeout = timeout;
this._timeoutRunner.updateTimeout(timeout);
} else {
if (!this.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
this.timeout = timeout;
this._timeoutRunner.updateTimeout(timeout);
}
if (!this.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
this.timeout = timeout;
this._timeoutRunner.updateTimeout(timeout);
}
}
class SkipError extends Error {
}
function titleForRunnable(runnable: RunnableDescription): string {
switch (runnable.type) {
case 'test':
return '';
case 'beforeAll':
case 'beforeEach':
case 'afterAll':
case 'afterEach':
return runnable.type + ' hook';
case 'teardown':
return 'fixtures teardown';
case 'skip':
case 'slow':
case 'fixme':
case 'fail':
return runnable.type + ' modifier';
}
}

19
packages/playwright-test/src/testType.ts
View file

@ -81,7 +81,7 @@ export class TestTypeImpl {
private _createTest(type: 'default' | 'only' | 'skip' | 'fixme', location: Location, title: string, fn: Function) {
throwIfRunningInsideJest();
const suite = this._ensureCurrentSuite(location, 'test()');
const test = new TestCase(title, fn, this, location);
const test = new TestCase('test', title, fn, nextOrdinalInFile(suite._requireFile), this, location);
test._requireFile = suite._requireFile;
suite._addTest(test);
@ -130,7 +130,15 @@ export class TestTypeImpl {
private _hook(name: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll', location: Location, fn: Function) {
const suite = this._ensureCurrentSuite(location, `test.${name}()`);
suite._hooks.push({ type: name, fn, location });
if (name === 'beforeAll' || name === 'afterAll') {
const sameTypeCount = suite.hooks.filter(hook => hook._type === name).length;
const suffix = sameTypeCount ? String(sameTypeCount) : '';
const hook = new TestCase(name, name + suffix, fn, nextOrdinalInFile(suite._requireFile), this, location);
hook._requireFile = suite._requireFile;
suite._addAllHook(hook);
} else {
suite._eachHooks.push({ type: name, fn, location });
}
}
private _configure(location: Location, options: { mode?: 'parallel' | 'serial' }) {
@ -243,4 +251,11 @@ function throwIfRunningInsideJest() {
}
}
const countByFile = new Map<string, number>();
function nextOrdinalInFile(file: string) {
const ordinalInFile = countByFile.get(file) || 0;
countByFile.set(file, ordinalInFile + 1);
return ordinalInFile;
}
export const rootTestType = new TestTypeImpl([]);

4
packages/playwright-test/src/types.ts
View file

@ -23,7 +23,7 @@ export type FixturesWithLocation = {
fixtures: Fixtures;
location: Location;
};
export type Annotation = { type: string, description?: string };
export type Annotations = { type: string, description?: string }[];
export interface TestStepInternal {
complete(error?: Error | TestError): void;
@ -33,3 +33,5 @@ export interface TestStepInternal {
forceNoParent: boolean;
location?: Location;
}
export type TestCaseType = 'beforeAll' | 'afterAll' | 'test';

2
packages/playwright-test/src/worker.ts
View file

@ -84,7 +84,7 @@ process.on('message', async message => {
}
if (message.method === 'run') {
const runPayload = message.params as RunPayload;
await workerRunner!.runTestGroup(runPayload);
await workerRunner!.run(runPayload);
}
});

409
packages/playwright-test/src/workerRunner.ts
View file

@ -18,15 +18,15 @@ import rimraf from 'rimraf';
import util from 'util';
import colors from 'colors/safe';
import { EventEmitter } from 'events';
import { serializeError } from './util';
import { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload, TeardownErrorsPayload } from './ipc';
import { serializeError, formatLocation } from './util';
import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload, TeardownErrorsPayload } from './ipc';
import { setCurrentTestInfo } from './globals';
import { Loader } from './loader';
import { Suite, TestCase } from './test';
import { Annotation, TestError, TestStepInternal } from './types';
import { Modifier, Suite, TestCase } from './test';
import { Annotations, TestError, TestInfo, TestStepInternal, WorkerInfo } from './types';
import { ProjectImpl } from './project';
import { FixtureRunner } from './fixtures';
import { ManualPromise, raceAgainstTimeout } from 'playwright-core/lib/utils/async';
import { raceAgainstTimeout } from 'playwright-core/lib/utils/async';
import { TestInfoImpl } from './testInfo';
const removeFolderAsync = util.promisify(rimraf);
@ -35,25 +35,15 @@ export class WorkerRunner extends EventEmitter {
private _params: WorkerInitParams;
private _loader!: Loader;
private _project!: ProjectImpl;
private _workerInfo!: WorkerInfo;
private _fixtureRunner: FixtureRunner;
// Accumulated fatal errors that cannot be attributed to a test.
private _failedTest: TestInfoImpl | undefined;
private _fatalErrors: TestError[] = [];
// Whether we should skip running remaining tests in the group because
// of a setup error, usually beforeAll hook.
private _skipRemainingTests = false;
// The stage of the full cleanup. Once "finished", we can safely stop running anything.
private _didRunFullCleanup = false;
// Whether the worker was requested to stop.
private _entries = new Map<string, TestEntry>();
private _isStopped = false;
// This promise resolves once the single "run test group" call finishes.
private _runFinished = new ManualPromise<void>();
private _runFinished = Promise.resolve();
_currentTest: TestInfoImpl | null = null;
// Dynamic annotations originated by modifiers with a callback, e.g. `test.skip(() => true)`.
private _extraSuiteAnnotations = new Map<Suite, Annotation[]>();
// Suites that had their beforeAll hooks, but not afterAll hooks executed.
// These suites still need afterAll hooks to be executed for the proper cleanup.
private _activeSuites = new Set<Suite>();
constructor(params: WorkerInitParams) {
super();
@ -110,7 +100,7 @@ export class WorkerRunner extends EventEmitter {
const isExpectError = (error instanceof Error) && !!(error as any).matcherResult;
const isCurrentTestExpectedToFail = this._currentTest?.expectedStatus === 'failed';
const shouldConsiderAsTestError = isExpectError || !isCurrentTestExpectedToFail;
if (this._currentTest && shouldConsiderAsTestError) {
if (this._currentTest && this._currentTest._test._type === 'test' && shouldConsiderAsTestError) {
this._currentTest._failWithError(serializeError(error), true /* isHardError */);
} else {
// No current test - fatal error.
@ -126,43 +116,102 @@ export class WorkerRunner extends EventEmitter {
this._loader = await Loader.deserialize(this._params.loader);
this._project = this._loader.projects()[this._params.projectIndex];
this._workerInfo = {
workerIndex: this._params.workerIndex,
parallelIndex: this._params.parallelIndex,
project: this._project.config,
config: this._loader.fullConfig(),
};
}
async runTestGroup(runPayload: RunPayload) {
this._runFinished = new ManualPromise<void>();
async run(runPayload: RunPayload) {
let runFinishedCallback = () => {};
this._runFinished = new Promise(f => runFinishedCallback = f);
try {
const entries = new Map(runPayload.entries.map(e => [ e.testId, e ]));
this._entries = new Map(runPayload.entries.map(e => [ e.testId, e ]));
await this._loadIfNeeded();
const fileSuite = await this._loader.loadTestFile(runPayload.file, 'worker');
const suite = this._project.cloneFileSuite(fileSuite, this._params.repeatEachIndex, test => {
if (!entries.has(test._id))
if (!this._entries.has(test._id))
return false;
return true;
});
if (suite) {
this._extraSuiteAnnotations = new Map();
this._activeSuites = new Set();
this._didRunFullCleanup = false;
const tests = suite.allTests().filter(test => entries.has(test._id));
for (let i = 0; i < tests.length; i++)
await this._runTest(tests[i], entries.get(tests[i]._id)!.retry, tests[i + 1]);
const firstPool = suite.allTests()[0]._pool!;
this._fixtureRunner.setPool(firstPool);
await this._runSuite(suite, []);
}
if (this._failedTest)
await this._teardownScopes();
} catch (e) {
// In theory, we should run above code without any errors.
// However, in the case we screwed up, or loadTestFile failed in the worker
// but not in the runner, let's do a fatal error.
this.unhandledError(e);
} finally {
if (this._failedTest) {
// Now that we did run all hooks and teared down scopes, we can
// report the failure, possibly with any error details revealed by teardown.
this.emit('testEnd', buildTestEndPayload(this._failedTest));
}
this._reportDone();
this._runFinished.resolve();
runFinishedCallback();
}
}
private async _runTest(test: TestCase, retry: number, nextTest: TestCase | undefined) {
// Do not run tests after full cleanup, because we are entirely done.
if (this._isStopped && this._didRunFullCleanup)
private async _runSuite(suite: Suite, annotations: Annotations) {
// When stopped, do not run a suite. But if we have started running the suite with hooks,
// always finish the hooks.
if (this._isStopped)
return;
annotations = annotations.concat(suite._annotations);
const allSkipped = suite.allTests().every(test => {
const runEntry = this._entries.get(test._id);
return !runEntry || test.expectedStatus === 'skipped';
});
if (allSkipped) {
// This avoids running beforeAll/afterAll hooks.
annotations.push({ type: 'skip' });
}
for (const beforeAllModifier of suite._modifiers) {
if (!this._fixtureRunner.dependsOnWorkerFixturesOnly(beforeAllModifier.fn, beforeAllModifier.location))
continue;
// TODO: separate timeout for beforeAll modifiers?
const result = await raceAgainstTimeout(() => this._fixtureRunner.resolveParametersAndRunFunction(beforeAllModifier.fn, this._workerInfo, undefined), this._project.config.timeout);
if (result.timedOut) {
this._fatalErrors.push(serializeError(new Error(`Timeout of ${this._project.config.timeout}ms exceeded while running ${beforeAllModifier.type} modifier\n at ${formatLocation(beforeAllModifier.location)}`)));
this.stop();
} else if (!!result.result) {
annotations.push({ type: beforeAllModifier.type, description: beforeAllModifier.description });
}
}
for (const hook of suite.hooks) {
if (hook._type !== 'beforeAll')
continue;
const firstTest = suite.allTests()[0];
await this._runTestOrAllHook(hook, annotations, this._entries.get(firstTest._id)?.retry || 0);
}
for (const entry of suite._entries) {
if (entry instanceof Suite) {
await this._runSuite(entry, annotations);
} else {
const runEntry = this._entries.get(entry._id);
if (runEntry && !this._isStopped)
await this._runTestOrAllHook(entry, annotations, runEntry.retry);
}
}
for (const hook of suite.hooks) {
if (hook._type !== 'afterAll')
continue;
await this._runTestOrAllHook(hook, annotations, 0);
}
}
private async _runTestOrAllHook(test: TestCase, annotations: Annotations, retry: number) {
let lastStepId = 0;
const testInfo = new TestInfoImpl(this._loader, this._params, test, retry, data => {
const stepId = `${data.category}@${data.title}@${++lastStepId}`;
@ -198,7 +247,16 @@ export class WorkerRunner extends EventEmitter {
return step;
});
const processAnnotation = (annotation: Annotation) => {
// Inherit test.setTimeout() from parent suites.
for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) {
if (suite._timeout !== undefined) {
testInfo.setTimeout(suite._timeout);
break;
}
}
// Process annotations defined on parent suites.
for (const annotation of annotations) {
testInfo.annotations.push(annotation);
switch (annotation.type) {
case 'fixme':
@ -213,35 +271,11 @@ export class WorkerRunner extends EventEmitter {
testInfo.setTimeout(testInfo.timeout * 3);
break;
}
};
if (!this._isStopped) {
// Update the fixture pool - it may differ between tests, but only in test-scoped fixtures.
this._fixtureRunner.setPool(test._pool!);
}
const suites = getSuites(test);
const reversedSuites = suites.slice().reverse();
// Inherit test.setTimeout() from parent suites, deepest has the priority.
for (const suite of reversedSuites) {
if (suite._timeout !== undefined) {
testInfo.setTimeout(suite._timeout);
break;
}
}
// Process existing annotations defined on parent suites.
for (const suite of suites) {
for (const annotation of suite._annotations)
processAnnotation(annotation);
const extraAnnotations = this._extraSuiteAnnotations.get(suite) || [];
for (const annotation of extraAnnotations)
processAnnotation(annotation);
}
this._currentTest = testInfo;
setCurrentTestInfo(testInfo);
this.emit('testBegin', buildTestBeginPayload(testInfo));
if (testInfo.expectedStatus === 'skipped') {
@ -250,137 +284,32 @@ export class WorkerRunner extends EventEmitter {
return;
}
// Assume beforeAll failed until we actually finish it successfully.
let didFailBeforeAll = true;
let shouldRunAfterEachHooks = false;
// Update the fixture pool - it may differ between tests, but only in test-scoped fixtures.
this._fixtureRunner.setPool(test._pool!);
await testInfo._runWithTimeout(async () => {
if (this._isStopped) {
// Getting here means that worker is requested to stop, but was not able to
// run full cleanup yet. Skip the test, but run the cleanup.
testInfo.status = 'skipped';
didFailBeforeAll = false;
return;
}
const beforeHooksStep = testInfo._addStep({
category: 'hook',
title: 'Before Hooks',
canHaveChildren: true,
forceNoParent: true
});
// Note: wrap all preparation steps together, because failure/skip in any of them
// prevents further setup and/or test from running.
const maybeError = await testInfo._runFn(async () => {
// Run "beforeAll" modifiers on parent suites, unless already run during previous tests.
for (const suite of suites) {
if (this._extraSuiteAnnotations.has(suite))
continue;
const extraAnnotations: Annotation[] = [];
this._extraSuiteAnnotations.set(suite, extraAnnotations);
await this._runModifiersForSuite(suite, testInfo, 'worker', extraAnnotations);
}
// Run "beforeAll" hooks, unless already run during previous tests.
for (const suite of suites)
await this._runBeforeAllHooksForSuite(suite, testInfo);
// Running "beforeAll" succeeded!
didFailBeforeAll = false;
// Run "beforeEach" modifiers.
for (const suite of suites)
await this._runModifiersForSuite(suite, testInfo, 'test');
// Run "beforeEach" hooks. Once started with "beforeEach", we must run all "afterEach" hooks as well.
shouldRunAfterEachHooks = true;
await this._runEachHooksForSuites(suites, 'beforeEach', testInfo);
// Setup fixtures required by the test.
testInfo._setCurrentRunnable({ type: 'test' });
const params = await this._fixtureRunner.resolveParametersForFunction(test.fn, testInfo);
beforeHooksStep.complete(); // Report fixture hooks step as completed.
// Now run the test itself.
const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]").
await fn(params, testInfo);
}, 'allowSkips');
beforeHooksStep.complete(maybeError); // Second complete is a no-op.
});
if (didFailBeforeAll) {
// This will inform dispatcher that we should not run more tests from this group
// because we had a beforeAll error.
// This behavior avoids getting the same common error for each test.
this._skipRemainingTests = true;
}
const afterHooksStep = testInfo._addStep({
category: 'hook',
title: 'After Hooks',
canHaveChildren: true,
forceNoParent: true
});
let firstAfterHooksError: TestError | undefined;
await testInfo._runWithTimeout(() => this._runTestWithBeforeHooks(test, testInfo));
if (testInfo.status === 'timedOut') {
// A timed-out test gets a full additional timeout to run after hooks.
testInfo._timeoutRunner.updateTimeout(testInfo.timeout, 0);
testInfo._timeoutRunner.resetTimeout(testInfo.timeout);
}
await testInfo._runWithTimeout(async () => {
// Note: do not wrap all teardown steps together, because failure in any of them
// does not prevent further teardown steps from running.
await testInfo._runWithTimeout(() => this._runAfterHooks(test, testInfo));
// Run "afterEach" hooks, unless we failed at beforeAll stage.
if (shouldRunAfterEachHooks) {
const afterEachError = await testInfo._runFn(() => this._runEachHooksForSuites(reversedSuites, 'afterEach', testInfo));
firstAfterHooksError = firstAfterHooksError || afterEachError;
}
// Run "afterAll" hooks for suites that are not shared with the next test.
const nextSuites = new Set(getSuites(nextTest));
for (const suite of reversedSuites) {
if (!nextSuites.has(suite)) {
const afterAllError = await this._runAfterAllHooksForSuite(suite, testInfo);
firstAfterHooksError = firstAfterHooksError || afterAllError;
}
}
// Teardown test-scoped fixtures.
testInfo._setCurrentRunnable({ type: 'teardown' });
const testScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test'));
firstAfterHooksError = firstAfterHooksError || testScopeError;
});
const isFailure = testInfo.status !== 'skipped' && testInfo.status !== testInfo.expectedStatus;
if (isFailure)
this._isStopped = true;
if (this._isStopped) {
// Run all remaining "afterAll" hooks and teardown all fixtures when worker is shutting down.
// Mark as "cleaned up" early to avoid running cleanup twice.
this._didRunFullCleanup = true;
// Give it more time for the full cleanup.
testInfo._timeoutRunner.updateTimeout(this._project.config.timeout, 0);
await testInfo._runWithTimeout(async () => {
for (const suite of reversedSuites) {
const afterAllError = await this._runAfterAllHooksForSuite(suite, testInfo);
firstAfterHooksError = firstAfterHooksError || afterAllError;
}
testInfo._setCurrentRunnable({ type: 'teardown', timeout: this._project.config.timeout });
const testScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test'));
firstAfterHooksError = firstAfterHooksError || testScopeError;
const workerScopeError = await testInfo._runFn(() => this._fixtureRunner.teardownScope('worker'));
firstAfterHooksError = firstAfterHooksError || workerScopeError;
});
}
afterHooksStep.complete(firstAfterHooksError);
this._currentTest = null;
setCurrentTestInfo(null);
this.emit('testEnd', buildTestEndPayload(testInfo));
const isFailure = testInfo.status !== 'skipped' && testInfo.status !== testInfo.expectedStatus;
if (isFailure) {
// Delay reporting testEnd result until after teardownScopes is done.
this._failedTest = testInfo;
if (test._type !== 'test') {
// beforeAll/afterAll hook failure skips any remaining tests in the worker.
this._fatalErrors.push(...testInfo.errors);
}
this.stop();
} else {
this.emit('testEnd', buildTestEndPayload(testInfo));
}
const preserveOutput = this._loader.fullConfig().preserveOutput === 'always' ||
(this._loader.fullConfig().preserveOutput === 'failures-only' && isFailure);
@ -388,63 +317,65 @@ export class WorkerRunner extends EventEmitter {
await removeFolderAsync(testInfo.outputDir).catch(e => {});
}
private async _runModifiersForSuite(suite: Suite, testInfo: TestInfoImpl, scope: 'worker' | 'test', extraAnnotations?: Annotation[]) {
for (const modifier of suite._modifiers) {
const actualScope = this._fixtureRunner.dependsOnWorkerFixturesOnly(modifier.fn, modifier.location) ? 'worker' : 'test';
if (actualScope !== scope)
continue;
testInfo._setCurrentRunnable({ type: modifier.type, location: modifier.location, timeout: scope === 'worker' ? this._project.config.timeout : undefined });
const result = await this._fixtureRunner.resolveParametersAndRunFunction(modifier.fn, testInfo);
if (result && extraAnnotations)
extraAnnotations.push({ type: modifier.type, description: modifier.description });
testInfo[modifier.type](!!result, modifier.description);
}
}
private async _runBeforeAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl) {
if (this._activeSuites.has(suite))
return;
this._activeSuites.add(suite);
let beforeAllError: Error | undefined;
for (const hook of suite._hooks) {
if (hook.type !== 'beforeAll')
continue;
try {
testInfo._setCurrentRunnable({ type: 'beforeAll', location: hook.location, timeout: this._project.config.timeout });
await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo);
} catch (e) {
// Always run all the hooks, and capture the first error.
beforeAllError = beforeAllError || e;
private async _runTestWithBeforeHooks(test: TestCase, testInfo: TestInfoImpl) {
const step = testInfo._addStep({
category: 'hook',
title: 'Before Hooks',
canHaveChildren: true,
forceNoParent: true
});
const maybeError = await testInfo._runFn(async () => {
if (test._type === 'test') {
const beforeEachModifiers: Modifier[] = [];
for (let s: Suite | undefined = test.parent; s; s = s.parent) {
const modifiers = s._modifiers.filter(modifier => !this._fixtureRunner.dependsOnWorkerFixturesOnly(modifier.fn, modifier.location));
beforeEachModifiers.push(...modifiers.reverse());
}
beforeEachModifiers.reverse();
for (const modifier of beforeEachModifiers) {
const result = await this._fixtureRunner.resolveParametersAndRunFunction(modifier.fn, this._workerInfo, testInfo);
testInfo[modifier.type](!!result, modifier.description!);
}
await this._runHooks(test.parent!, 'beforeEach', testInfo);
}
}
if (beforeAllError)
throw beforeAllError;
const params = await this._fixtureRunner.resolveParametersForFunction(test.fn, this._workerInfo, testInfo);
step.complete(); // Report fixture hooks step as completed.
const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]").
await fn(params, testInfo);
}, 'allowSkips');
step.complete(maybeError); // Second complete is a no-op.
}
private async _runAfterAllHooksForSuite(suite: Suite, testInfo: TestInfoImpl) {
if (!this._activeSuites.has(suite))
return;
this._activeSuites.delete(suite);
let firstError: TestError | undefined;
for (const hook of suite._hooks) {
if (hook.type !== 'afterAll')
continue;
const afterAllError = await testInfo._runFn(async () => {
testInfo._setCurrentRunnable({ type: 'afterAll', location: hook.location, timeout: this._project.config.timeout });
await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo);
});
firstError = firstError || afterAllError;
}
return firstError;
private async _runAfterHooks(test: TestCase, testInfo: TestInfoImpl) {
const step = testInfo._addStep({
category: 'hook',
title: 'After Hooks',
canHaveChildren: true,
forceNoParent: true
});
let teardownError1: TestError | undefined;
if (test._type === 'test')
teardownError1 = await testInfo._runFn(() => this._runHooks(test.parent!, 'afterEach', testInfo));
// Continue teardown even after the failure.
const teardownError2 = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test'));
step.complete(teardownError1 || teardownError2);
}
private async _runEachHooksForSuites(suites: Suite[], type: 'beforeEach' | 'afterEach', testInfo: TestInfoImpl) {
const hooks = suites.map(suite => suite._hooks.filter(hook => hook.type === type)).flat();
private async _runHooks(suite: Suite, type: 'beforeEach' | 'afterEach', testInfo: TestInfo) {
const all = [];
for (let s: Suite | undefined = suite; s; s = s.parent) {
const funcs = s._eachHooks.filter(e => e.type === type).map(e => e.fn);
all.push(...funcs.reverse());
}
if (type === 'beforeEach')
all.reverse();
let error: Error | undefined;
for (const hook of hooks) {
for (const hook of all) {
try {
testInfo._setCurrentRunnable({ type, location: hook.location });
await this._fixtureRunner.resolveParametersAndRunFunction(hook.fn, testInfo);
await this._fixtureRunner.resolveParametersAndRunFunction(hook, this._workerInfo, testInfo);
} catch (e) {
// Always run all the hooks, and capture the first error.
error = error || e;
@ -455,10 +386,10 @@ export class WorkerRunner extends EventEmitter {
}
private _reportDone() {
const donePayload: DonePayload = { fatalErrors: this._fatalErrors, skipRemaining: this._skipRemainingTests };
const donePayload: DonePayload = { fatalErrors: this._fatalErrors };
this.emit('done', donePayload);
this._fatalErrors = [];
this._skipRemainingTests = false;
this._failedTest = undefined;
}
}
@ -486,11 +417,3 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {
}))
};
}
function getSuites(test: TestCase | undefined): Suite[] {
const suites: Suite[] = [];
for (let suite: Suite | undefined = test?.parent; suite; suite = suite.parent)
suites.push(suite);
suites.reverse(); // Put root suite first.
return suites;
}

21
tests/playwright-test/fixture-errors.spec.ts
View file

@ -453,26 +453,7 @@ test('should not report fixture teardown error twice', async ({ runInlineTest })
expect(result.failed).toBe(1);
expect(result.output).toContain('Error: Oh my error');
expect(stripAnsi(result.output)).toContain(`throw new Error('Oh my error')`);
expect(countTimes(stripAnsi(result.output), 'Oh my error')).toBe(2);
});
test('should not report fixture teardown timeout twice', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
const test = pwt.test.extend({
fixture: async ({ }, use) => {
await use();
await new Promise(() => {});
},
});
test('good', async ({ fixture }) => {
});
`,
}, { reporter: 'list', timeout: 1000 });
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('while shutting down environment');
expect(countTimes(result.output, 'while shutting down environment')).toBe(1);
expect(countTimes(result.output, 'Oh my error')).toBe(2);
});
test('should handle fixture teardown error after test timeout and continue', async ({ runInlineTest }) => {

10
tests/playwright-test/fixtures.spec.ts
View file

@ -316,23 +316,23 @@ test('automatic fixtures should work', async ({ runInlineTest }) => {
});
test.beforeEach(async ({}) => {
expect(counterWorker).toBe(1);
expect(counterTest === 1 || counterTest === 2).toBe(true);
expect(counterTest === 2 || counterTest === 3).toBe(true);
});
test('test 1', async ({}) => {
expect(counterWorker).toBe(1);
expect(counterTest).toBe(1);
expect(counterTest).toBe(2);
});
test('test 2', async ({}) => {
expect(counterWorker).toBe(1);
expect(counterTest).toBe(2);
expect(counterTest).toBe(3);
});
test.afterEach(async ({}) => {
expect(counterWorker).toBe(1);
expect(counterTest === 1 || counterTest === 2).toBe(true);
expect(counterTest === 2 || counterTest === 3).toBe(true);
});
test.afterAll(async ({}) => {
expect(counterWorker).toBe(1);
expect(counterTest).toBe(2);
expect(counterTest).toBe(4);
});
`
});

192
tests/playwright-test/hooks.spec.ts
View file

@ -63,10 +63,14 @@ test('hooks should work with fixtures', async ({ runInlineTest }) => {
'+w',
'+t',
'beforeAll-17-42',
'beforeEach-17-42',
'test-17-42',
'afterEach-17-42',
'afterAll-17-42',
'-t',
'+t',
'beforeEach-17-43',
'test-17-43',
'afterEach-17-43',
'-t',
'+t',
'afterAll-17-44',
'-t',
'+t',
]);
@ -92,11 +96,11 @@ test('afterEach failure should not prevent other hooks and fixtures teardown', a
const { test } = require('./helper');
test.describe('suite', () => {
test.afterEach(async () => {
console.log('afterEach2');
throw new Error('afterEach2');
console.log('afterEach1');
});
test.afterEach(async () => {
console.log('afterEach1');
console.log('afterEach2');
throw new Error('afterEach2');
});
test('one', async ({foo}) => {
console.log('test');
@ -309,7 +313,7 @@ test('beforeAll hook should get retry index of the first test', async ({ runInli
]);
});
test('afterAll exception should fail the test', async ({ runInlineTest }) => {
test('afterAll exception should fail the run', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
@ -321,8 +325,7 @@ test('afterAll exception should fail the test', async ({ runInlineTest }) => {
`,
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(1);
expect(result.passed).toBe(1);
expect(result.output).toContain('From the afterAll');
});
@ -367,17 +370,13 @@ test('beforeAll failure should prevent the test, but not afterAll', async ({ run
test.afterAll(() => {
console.log('\\n%%afterAll');
});
test('failed', () => {
console.log('\\n%%test1');
});
test('skipped', () => {
console.log('\\n%%test2');
console.log('\\n%%test');
});
`,
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.skipped).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%beforeAll',
'%%afterAll',
@ -455,7 +454,7 @@ test('afterAll error should not mask beforeAll', async ({ runInlineTest }) => {
expect(result.output).toContain('from beforeAll');
});
test('beforeAll timeout should be reported and prevent more tests', async ({ runInlineTest }) => {
test('beforeAll timeout should be reported', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
@ -466,62 +465,41 @@ test('beforeAll timeout should be reported and prevent more tests', async ({ run
test.afterAll(() => {
console.log('\\n%%afterAll');
});
test('failed', () => {
console.log('\\n%%test1');
});
test('skipped', () => {
console.log('\\n%%test2');
console.log('\\n%%test');
});
`,
}, { timeout: 1000 });
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.skipped).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%beforeAll',
'%%afterAll',
]);
expect(result.output).toContain('Timeout of 1000ms exceeded in beforeAll hook.');
expect(result.output).toContain(`a.test.js:6:12`);
expect(stripAnsi(result.output)).toContain(`> 6 | test.beforeAll(async () => {`);
});
test('afterAll timeout should be reported, run other afterAll hooks, and continue testing', async ({ runInlineTest }, testInfo) => {
test('afterAll timeout should be reported', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
test.describe('suite', () => {
test.afterAll(async () => {
console.log('\\n%%afterAll1');
await new Promise(f => setTimeout(f, 5000));
});
test('runs', () => {
test.setTimeout(2000);
console.log('\\n%%test1');
});
});
test.afterAll(async () => {
console.log('\\n%%afterAll2');
console.log('\\n%%afterAll');
await new Promise(f => setTimeout(f, 5000));
});
test('does not run', () => {
console.log('\\n%%test2');
test('runs', () => {
console.log('\\n%%test');
});
`,
}, { timeout: 1000 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(1);
expect(result.failed).toBe(1);
expect(result.skipped).toBe(0);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%test1',
'%%afterAll1',
'%%afterAll2',
'%%test2',
'%%afterAll2',
'%%test',
'%%afterAll',
]);
expect(result.output).toContain('Timeout of 1000ms exceeded in afterAll hook.');
expect(result.output).toContain(`a.test.js:7:14`);
expect(stripAnsi(result.output)).toContain(`> 7 | test.afterAll(async () => {`);
expect(result.output).toContain(`at a.test.js:6:12`);
});
test('beforeAll and afterAll timeouts at the same time should be reported', async ({ runInlineTest }) => {
@ -628,127 +606,13 @@ test('should not hang and report results when worker process suddenly exits duri
const result = await runInlineTest({
'a.spec.js': `
const { test } = pwt;
test('failing due to afterall', () => {});
test('passed', () => {});
test.afterAll(() => { process.exit(0); });
`
}, { reporter: 'line' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(1);
expect(result.passed).toBe(1);
expect(result.output).toContain('Worker process exited unexpectedly');
expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 › failing due to afterall');
});
test('unhandled rejection during beforeAll should be reported and prevent more tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
test.beforeAll(async () => {
console.log('\\n%%beforeAll');
Promise.resolve().then(() => {
throw new Error('Oh my');
});
await new Promise(f => setTimeout(f, 100));
});
test.afterAll(() => {
console.log('\\n%%afterAll');
});
test('failed', () => {
console.log('\\n%%test1');
});
test('skipped', () => {
console.log('\\n%%test2');
});
`,
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.skipped).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%beforeAll',
'%%afterAll',
]);
expect(result.output).toContain('Error: Oh my');
expect(stripAnsi(result.output)).toContain(`> 9 | throw new Error('Oh my');`);
});
test('beforeAll and afterAll should have a separate timeout', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
test.beforeAll(async () => {
console.log('\\n%%beforeAll');
await new Promise(f => setTimeout(f, 300));
});
test.beforeAll(async () => {
console.log('\\n%%beforeAll2');
await new Promise(f => setTimeout(f, 300));
});
test('passed', async () => {
console.log('\\n%%test');
await new Promise(f => setTimeout(f, 300));
});
test.afterAll(async () => {
console.log('\\n%%afterAll');
await new Promise(f => setTimeout(f, 300));
});
test.afterAll(async () => {
console.log('\\n%%afterAll2');
await new Promise(f => setTimeout(f, 300));
});
`,
}, { timeout: '500' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%beforeAll',
'%%beforeAll2',
'%%test',
'%%afterAll',
'%%afterAll2',
]);
});
test('test.setTimeout should work separately in beforeAll', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
test.beforeAll(async () => {
console.log('\\n%%beforeAll');
test.setTimeout(100);
});
test('passed', async () => {
console.log('\\n%%test');
await new Promise(f => setTimeout(f, 800));
});
`,
}, { timeout: '1000' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%beforeAll',
'%%test',
]);
});
test('test.setTimeout should work separately in afterAll', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test } = pwt;
test('passed', async () => {
console.log('\\n%%test');
});
test.afterAll(async () => {
console.log('\\n%%afterAll');
test.setTimeout(1000);
await new Promise(f => setTimeout(f, 800));
});
`,
}, { timeout: '100' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%test',
'%%afterAll',
]);
expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 › passed');
expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:7:12 › afterAll');
});

49
tests/playwright-test/loader.spec.ts
View file

@ -407,52 +407,3 @@ test('should filter stack even without default Error.prepareStackTrace', async (
expect(stackLines.length).toBe(1);
});
test('should work with cross-imports - 1', async ({ runInlineTest }) => {
const result = await runInlineTest({
'test1.spec.ts': `
const { test } = pwt;
test('test 1', async ({}) => {
await new Promise(x => setTimeout(x, 500));
console.log('running TEST-1');
});
`,
'test2.spec.ts': `
import * as _ from './test1.spec';
const { test } = pwt;
test('test 2', async ({}) => {
await new Promise(x => setTimeout(x, 500));
console.log('running TEST-2');
});
`
}, { workers: 2 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.failed).toBe(0);
expect(result.output).toContain('TEST-1');
expect(result.output).toContain('TEST-2');
});
test('should work with cross-imports - 2', async ({ runInlineTest }) => {
const result = await runInlineTest({
'test1.spec.ts': `
const { test } = pwt;
import * as _ from './test2.spec';
test('test 1', async ({}) => {
await new Promise(x => setTimeout(x, 500));
console.log('running TEST-1');
});
`,
'test2.spec.ts': `
const { test } = pwt;
test('test 2', async ({}) => {
await new Promise(x => setTimeout(x, 500));
console.log('running TEST-2');
});
`
}, { workers: 2, reporter: 'list' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.failed).toBe(0);
expect(result.output).toContain('TEST-1');
expect(result.output).toContain('TEST-2');
});

11
tests/playwright-test/playwright.artifacts.spec.ts
View file

@ -46,7 +46,6 @@ const testFiles = {
});
test.afterAll(async () => {
await page.setContent('Reset!');
await page.close();
});
@ -146,6 +145,10 @@ test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => {
' test-failed-1.png',
'artifacts-persistent-passing',
' test-finished-1.png',
'artifacts-shared-afterAll-worker0',
' test-finished-1.png',
'artifacts-shared-beforeAll-worker0',
' test-finished-1.png',
'artifacts-shared-shared-failing',
' test-failed-1.png',
'artifacts-shared-shared-passing',
@ -211,6 +214,10 @@ test('should work with trace: on', async ({ runInlineTest }, testInfo) => {
' trace.zip',
'artifacts-persistent-passing',
' trace.zip',
'artifacts-shared-afterAll-worker0',
' trace.zip',
'artifacts-shared-beforeAll-worker0',
' trace.zip',
'artifacts-shared-shared-failing',
' trace.zip',
'artifacts-shared-shared-passing',
@ -270,6 +277,8 @@ test('should work with trace: on-first-retry', async ({ runInlineTest }, testInf
' trace.zip',
'artifacts-persistent-failing-retry1',
' trace.zip',
'artifacts-shared-beforeAll-worker1-retry1',
' trace.zip',
'artifacts-shared-shared-failing-retry1',
' trace.zip',
'artifacts-two-contexts-failing-retry1',

12
tests/playwright-test/reporter.spec.ts
View file

@ -343,11 +343,15 @@ test('should report api steps', async ({ runInlineTest }) => {
`%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"},{\"title\":\"browserContext.close\",\"category\":\"pw:api\"}]}`,
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`,
`%% begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"browser.newPage\",\"category\":\"pw:api\"},{\"title\":\"page.setContent\",\"category\":\"pw:api\"}]}`,
`%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`,
`%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
@ -357,9 +361,13 @@ test('should report api steps', async ({ runInlineTest }) => {
`%% begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`,
`%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"page.close\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"page.close\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}`,
`%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
]);
});

34
tests/playwright-test/test-modifiers.spec.ts
View file

@ -330,38 +330,6 @@ test('modifier timeout should be reported', async ({ runInlineTest }) => {
}, { timeout: 2000 });
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Timeout of 2000ms exceeded in skip modifier.');
expect(result.output).toContain('Error: Timeout of 2000ms exceeded while running skip modifier');
expect(stripAnsi(result.output)).toContain('6 | test.skip(async () => new Promise(() => {}));');
});
test('should not run hooks if modifier throws', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
const { test } = pwt;
test.skip(() => {
console.log('%%modifier');
throw new Error('Oh my');
});
test.beforeAll(() => {
console.log('%%beforeEach');
});
test.beforeEach(() => {
console.log('%%beforeEach');
});
test.afterEach(() => {
console.log('%%afterEach');
});
test.afterAll(() => {
console.log('%%beforeEach');
});
test('skipped1', () => {
console.log('%%skipped1');
});
`,
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%modifier',
]);
});

42
tests/playwright-test/test-output-dir.spec.ts
View file

@ -83,27 +83,53 @@ test('should include repeat token', async ({ runInlineTest }) => {
expect(result.passed).toBe(3);
});
test('should be unique for beforeAll hook from different workers', async ({ runInlineTest }, testInfo) => {
test('should be unique for beforeAll and afterAll hooks', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'a.spec.js': `
const { test } = pwt;
test.beforeAll(({}, testInfo) => {
console.log('\\n%%' + testInfo.outputDir);
});
test('fails', ({}, testInfo) => {
expect(1).toBe(2);
test.beforeAll(({}, testInfo) => {
console.log('\\n%%' + testInfo.outputDir);
});
test('passes', ({}, testInfo) => {
test.afterAll(({}, testInfo) => {
console.log('\\n%%' + testInfo.outputDir);
});
test.afterAll(({}, testInfo) => {
console.log('\\n%%' + testInfo.outputDir);
});
test.describe('suite', () => {
test.beforeAll(({}, testInfo) => {
console.log('\\n%%' + testInfo.outputDir);
});
test.afterAll(({}, testInfo) => {
console.log('\\n%%' + testInfo.outputDir);
});
test('fails', ({}, testInfo) => {
expect(1).toBe(2);
});
test('passes', ({}, testInfo) => {
});
});
`
}, { retries: '1' });
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(1);
expect(result.failed).toBe(1);
expect(result.output.split('\n').filter(x => x.startsWith('%%'))).toEqual([
`%%${testInfo.outputPath('test-results', 'a-fails')}`,
`%%${testInfo.outputPath('test-results', 'a-fails-retry1')}`,
`%%${testInfo.outputPath('test-results', 'a-passes')}`,
`%%${testInfo.outputPath('test-results', 'a-beforeAll-worker0')}`,
`%%${testInfo.outputPath('test-results', 'a-beforeAll1-worker0')}`,
`%%${testInfo.outputPath('test-results', 'a-suite-beforeAll-worker0')}`,
`%%${testInfo.outputPath('test-results', 'a-suite-afterAll-worker0')}`,
`%%${testInfo.outputPath('test-results', 'a-afterAll-worker0')}`,
`%%${testInfo.outputPath('test-results', 'a-afterAll1-worker0')}`,
`%%${testInfo.outputPath('test-results', 'a-beforeAll-worker1')}`,
`%%${testInfo.outputPath('test-results', 'a-beforeAll1-worker1')}`,
`%%${testInfo.outputPath('test-results', 'a-suite-beforeAll-worker1')}`,
`%%${testInfo.outputPath('test-results', 'a-suite-afterAll-worker1')}`,
`%%${testInfo.outputPath('test-results', 'a-afterAll-worker1')}`,
`%%${testInfo.outputPath('test-results', 'a-afterAll1-worker1')}`,
]);
});