/** * Copyright Microsoft Corporation. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import fs from 'fs'; import path from 'path'; import rimraf from 'rimraf'; import util from 'util'; import { EventEmitter } from 'events'; import { monotonicTime, DeadlineRunner, raceAgainstDeadline, serializeError } from './util'; import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams } from './ipc'; import { setCurrentTestInfo } from './globals'; import { Loader } from './loader'; import { Spec, Suite, Test } from './test'; import { TestInfo, WorkerInfo } from './types'; import { ProjectImpl } from './project'; import { FixtureRunner } from './fixtures'; const removeFolderAsync = util.promisify(rimraf); export class WorkerRunner extends EventEmitter { private _params: WorkerInitParams; private _loader!: Loader; private _project!: ProjectImpl; private _workerInfo!: WorkerInfo; private _projectNamePathSegment = ''; private _uniqueProjectNamePathSegment = ''; private _fixtureRunner: FixtureRunner; private _failedTestId: string | undefined; private _fatalError: any | undefined; private _entries = new Map(); private _remaining = new Map(); private _isStopped: any; _currentTest: { testId: string, testInfo: TestInfo } | null = null; constructor(params: WorkerInitParams) { super(); this._params = params; this._fixtureRunner = new FixtureRunner(); } stop() { this._isStopped = true; this._setCurrentTest(null); } async cleanup() { // TODO: separate timeout for teardown? const result = await raceAgainstDeadline((async () => { await this._fixtureRunner.teardownScope('test'); await this._fixtureRunner.teardownScope('worker'); })(), this._deadline()); if (result.timedOut) throw new Error(`Timeout of ${this._project.config.timeout}ms exceeded while shutting down environment`); } unhandledError(error: Error | any) { if (this._isStopped) return; if (this._currentTest) { this._currentTest.testInfo.status = 'failed'; this._currentTest.testInfo.error = serializeError(error); this._failedTestId = this._currentTest.testId; this.emit('testEnd', buildTestEndPayload(this._currentTest.testId, this._currentTest.testInfo)); } else { // No current test - fatal error. this._fatalError = serializeError(error); } this._reportDoneAndStop(); } private _deadline() { return this._project.config.timeout ? monotonicTime() + this._project.config.timeout : undefined; } private _loadIfNeeded() { if (this._loader) return; this._loader = Loader.deserialize(this._params.loader); this._project = this._loader.projects()[this._params.projectIndex]; this._projectNamePathSegment = sanitizeForFilePath(this._project.config.name); const sameName = this._loader.projects().filter(project => project.config.name === this._project.config.name); if (sameName.length > 1) this._uniqueProjectNamePathSegment = this._project.config.name + (sameName.indexOf(this._project) + 1); else this._uniqueProjectNamePathSegment = this._project.config.name; this._uniqueProjectNamePathSegment = sanitizeForFilePath(this._uniqueProjectNamePathSegment); this._workerInfo = { workerIndex: this._params.workerIndex, project: this._project.config, config: this._loader.fullConfig(), }; } async run(runPayload: RunPayload) { this._entries = new Map(runPayload.entries.map(e => [ e.testId, e ])); this._remaining = new Map(runPayload.entries.map(e => [ e.testId, e ])); this._loadIfNeeded(); const fileSuite = this._loader.loadTestFile(runPayload.file); let anySpec: Spec | undefined; fileSuite.findSpec(spec => { const test = this._project.generateTests(spec, this._params.repeatEachIndex)[0]; if (this._entries.has(test._id)) anySpec = spec; }); if (!anySpec) { this._reportDone(); return; } this._fixtureRunner.setPool(this._project.buildPool(anySpec)); await this._runSuite(fileSuite); if (this._isStopped) return; this._reportDone(); } private async _runSuite(suite: Suite) { if (this._isStopped) return; const skipHooks = !this._hasTestsToRun(suite); for (const hook of suite._hooks) { if (hook.type !== 'beforeAll' || skipHooks) continue; if (this._isStopped) return; // TODO: separate timeout for beforeAll? const result = await raceAgainstDeadline(this._fixtureRunner.resolveParametersAndRunHookOrTest(hook.fn, 'worker', this._workerInfo), this._deadline()); if (result.timedOut) { this._fatalError = serializeError(new Error(`Timeout of ${this._project.config.timeout}ms exceeded while running beforeAll hook`)); this._reportDoneAndStop(); } } for (const entry of suite._entries) { if (entry instanceof Suite) await this._runSuite(entry); else await this._runSpec(entry); } for (const hook of suite._hooks) { if (hook.type !== 'afterAll' || skipHooks) continue; if (this._isStopped) return; // TODO: separate timeout for afterAll? const result = await raceAgainstDeadline(this._fixtureRunner.resolveParametersAndRunHookOrTest(hook.fn, 'worker', this._workerInfo), this._deadline()); if (result.timedOut) { this._fatalError = serializeError(new Error(`Timeout of ${this._project.config.timeout}ms exceeded while running afterAll hook`)); this._reportDoneAndStop(); } } } private async _runSpec(spec: Spec) { if (this._isStopped) return; const test = spec.tests[0]; const entry = this._entries.get(test._id); if (!entry) return; this._remaining.delete(test._id); const startTime = monotonicTime(); let deadlineRunner: DeadlineRunner | undefined; const testId = test._id; const baseOutputDir = (() => { const relativeTestFilePath = path.relative(this._project.config.testDir, spec.file.replace(/\.(spec|test)\.(js|ts)/, '')); const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-'); let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(spec.title); if (this._uniqueProjectNamePathSegment) testOutputDir += '-' + this._uniqueProjectNamePathSegment; if (entry.retry) testOutputDir += '-retry' + entry.retry; if (this._params.repeatEachIndex) testOutputDir += '-repeat' + this._params.repeatEachIndex; return path.join(this._project.config.outputDir, testOutputDir); })(); const testInfo: TestInfo = { ...this._workerInfo, title: spec.title, file: spec.file, line: spec.line, column: spec.column, fn: spec.fn, repeatEachIndex: this._params.repeatEachIndex, retry: entry.retry, expectedStatus: 'passed', annotations: [], duration: 0, status: 'passed', stdout: [], stderr: [], timeout: this._project.config.timeout, snapshotSuffix: '', outputDir: baseOutputDir, outputPath: (...pathSegments: string[]): string => { fs.mkdirSync(baseOutputDir, { recursive: true }); return path.join(baseOutputDir, ...pathSegments); }, snapshotPath: (snapshotName: string): string => { let suffix = ''; if (this._projectNamePathSegment) suffix += '-' + this._projectNamePathSegment; if (testInfo.snapshotSuffix) suffix += '-' + testInfo.snapshotSuffix; if (suffix) { const ext = path.extname(snapshotName); if (ext) snapshotName = snapshotName.substring(0, snapshotName.length - ext.length) + suffix + ext; else snapshotName += suffix; } return path.join(spec.file + '-snapshots', snapshotName); }, skip: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'skip', args), fixme: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fixme', args), fail: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fail', args), slow: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'slow', args), setTimeout: (timeout: number) => { testInfo.timeout = timeout; if (deadlineRunner) deadlineRunner.setDeadline(deadline()); }, }; this._setCurrentTest({ testInfo, testId }); const deadline = () => { return testInfo.timeout ? startTime + testInfo.timeout : undefined; }; this.emit('testBegin', buildTestBeginPayload(testId, testInfo)); if (testInfo.expectedStatus === 'skipped') { testInfo.status = 'skipped'; this.emit('testEnd', buildTestEndPayload(testId, testInfo)); return; } // Update the fixture pool - it may differ between tests, but only in test-scoped fixtures. this._fixtureRunner.setPool(this._project.buildPool(spec)); deadlineRunner = new DeadlineRunner(this._runTestWithBeforeHooks(test, testInfo), deadline()); const result = await deadlineRunner.result; // Do not overwrite test failure upon hook timeout. if (result.timedOut && testInfo.status === 'passed') testInfo.status = 'timedOut'; if (this._isStopped) return; if (!result.timedOut) { deadlineRunner = new DeadlineRunner(this._runAfterHooks(test, testInfo), deadline()); deadlineRunner.setDeadline(deadline()); const hooksResult = await deadlineRunner.result; // Do not overwrite test failure upon hook timeout. if (hooksResult.timedOut && testInfo.status === 'passed') testInfo.status = 'timedOut'; } else { // A timed-out test gets a full additional timeout to run after hooks. const newDeadline = this._deadline(); deadlineRunner = new DeadlineRunner(this._runAfterHooks(test, testInfo), newDeadline); await deadlineRunner.result; } if (this._isStopped) return; testInfo.duration = monotonicTime() - startTime; this.emit('testEnd', buildTestEndPayload(testId, testInfo)); const isFailure = testInfo.status === 'timedOut' || (testInfo.status === 'failed' && testInfo.expectedStatus !== 'failed'); const preserveOutput = this._loader.fullConfig().preserveOutput === 'always' || (this._loader.fullConfig().preserveOutput === 'failures-only' && isFailure); if (!preserveOutput) await removeFolderAsync(testInfo.outputDir).catch(e => {}); if (testInfo.status !== 'passed') { this._failedTestId = testId; this._reportDoneAndStop(); } this._setCurrentTest(null); } private _setCurrentTest(currentTest: { testId: string, testInfo: TestInfo} | null) { this._currentTest = currentTest; setCurrentTestInfo(currentTest ? currentTest.testInfo : null); } private async _runTestWithBeforeHooks(test: Test, testInfo: TestInfo) { try { await this._runHooks(test.spec.parent!, 'beforeEach', testInfo); } catch (error) { if (error instanceof SkipError) { if (testInfo.status === 'passed') testInfo.status = 'skipped'; } else { testInfo.status = 'failed'; testInfo.error = serializeError(error); } // Continue running afterEach hooks even after the failure. } // Do not run the test when beforeEach hook fails. if (this._isStopped || testInfo.status === 'failed' || testInfo.status === 'skipped') return; try { await this._fixtureRunner.resolveParametersAndRunHookOrTest(test.spec.fn, 'test', testInfo); } catch (error) { if (error instanceof SkipError) { if (testInfo.status === 'passed') testInfo.status = 'skipped'; } else { // We might fail after the timeout, e.g. due to fixture teardown. // Do not overwrite the timeout status with this error. if (testInfo.status === 'passed') { testInfo.status = 'failed'; testInfo.error = serializeError(error); } } } } private async _runAfterHooks(test: Test, testInfo: TestInfo) { try { await this._runHooks(test.spec.parent!, 'afterEach', testInfo); } catch (error) { // Do not overwrite test failure error. if (!(error instanceof SkipError) && testInfo.status === 'passed') { testInfo.status = 'failed'; testInfo.error = serializeError(error); // Continue running even after the failure. } } try { await this._fixtureRunner.teardownScope('test'); } catch (error) { // Do not overwrite test failure error. if (testInfo.status === 'passed') { testInfo.status = 'failed'; testInfo.error = serializeError(error); } } } private async _runHooks(suite: Suite, type: 'beforeEach' | 'afterEach', testInfo: TestInfo) { if (this._isStopped) return; const all = []; for (let s: Suite | undefined = suite; s; s = s.parent) { const funcs = s._hooks.filter(e => e.type === type).map(e => e.fn); all.push(...funcs.reverse()); } if (type === 'beforeEach') all.reverse(); let error: Error | undefined; for (const hook of all) { try { await this._fixtureRunner.resolveParametersAndRunHookOrTest(hook, 'test', testInfo); } catch (e) { // Always run all the hooks, and capture the first error. error = error || e; } } if (error) throw error; } private _reportDone() { const donePayload: DonePayload = { failedTestId: this._failedTestId, fatalError: this._fatalError, remaining: [...this._remaining.values()], }; this.emit('done', donePayload); } private _reportDoneAndStop() { if (this._isStopped) return; this._reportDone(); this.stop(); } private _hasTestsToRun(suite: Suite): boolean { return suite.findSpec(spec => { const entry = this._entries.get(spec.tests[0]._id); return !!entry; }); } } function buildTestBeginPayload(testId: string, testInfo: TestInfo): TestBeginPayload { return { testId, workerIndex: testInfo.workerIndex }; } function buildTestEndPayload(testId: string, testInfo: TestInfo): TestEndPayload { return { testId, duration: testInfo.duration, status: testInfo.status!, error: testInfo.error, expectedStatus: testInfo.expectedStatus, annotations: testInfo.annotations, timeout: testInfo.timeout, }; } function modifier(testInfo: TestInfo, type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) { if (modifierArgs.length >= 1 && !modifierArgs[0]) return; const description = modifierArgs[1]; testInfo.annotations.push({ type, description }); if (type === 'slow') { testInfo.setTimeout(testInfo.timeout * 3); } else if (type === 'skip' || type === 'fixme') { testInfo.expectedStatus = 'skipped'; throw new SkipError('Test is skipped: ' + (description || '')); } else if (type === 'fail') { if (testInfo.expectedStatus !== 'skipped') testInfo.expectedStatus = 'failed'; } } class SkipError extends Error { } function sanitizeForFilePath(s: string) { return s.replace(/[^\w\d]+/g, '-'); }