Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore: run global setup before onBegin (#20285)

Browse source
This commit is contained in:
Pavel Feldman 2023-01-23 17:44:23 -08:00 committed by GitHub
parent 0ec1d5d452
commit 147bb6b292
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 163 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
docs/src/test-reporter-api/class-reporter.md
View file

@ -79,11 +79,13 @@ export default defineConfig({
```
Here is a typical order of reporter calls:
* [`method: Reporter.onConfigure`] is called once config has been resolved.
* [`method: Reporter.onBegin`] is called once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite].
* [`method: Reporter.onTestBegin`] is called for each test run. It is given a [TestCase] that is executed, and a [TestResult] that is almost empty. Test result will be populated while the test runs (for example, with steps and stdio) and will get final `status` once the test finishes.
* [`method: Reporter.onStepBegin`] and [`method: Reporter.onStepEnd`] are called for each executed step inside the test. When steps are executed, test run has not finished yet.
* [`method: Reporter.onTestEnd`] is called when test run has finished. By this time, [TestResult] is complete and you can use [`property: TestResult.status`], [`property: TestResult.error`] and more.
* [`method: Reporter.onEnd`] is called once after all tests that should run had finished.
* [`method: Reporter.onExit`] is called before test runner exits.
Additionally, [`method: Reporter.onStdOut`] and [`method: Reporter.onStdErr`] are called when standard output is produced in the worker process, possibly during a test execution,
and [`method: Reporter.onError`] is called when something went wrong outside of the test execution.
@ -107,7 +109,16 @@ Resolved configuration.
The root suite that contains all projects, files and test cases.
## optional method: Reporter.onConfigure
* since: v1.30
Called once config is resolved.
### param: Reporter.onConfigure.config
* since: v1.30
- `config` <[TestConfig]>
Resolved configuration.
## optional async method: Reporter.onEnd
* since: v1.10
@ -125,8 +136,10 @@ Result of the full test run.
* `'timedout'` - The [`property: TestConfig.globalTimeout`] has been reached.
* `'interrupted'` - Interrupted by the user.
## optional method: Reporter.onExit
* since: v1.30
Called before test runner exits.
## optional method: Reporter.onError
* since: v1.10

4
packages/playwright-test/src/plugins/dockerPlugin.ts
View file

@ -15,14 +15,14 @@
*/
import type { TestRunnerPlugin } from '.';
import type { FullConfig, Reporter, Suite } from '../../types/testReporter';
import type { FullConfig, Reporter } from '../../types/testReporter';
import { colors } from 'playwright-core/lib/utilsBundle';
import { checkDockerEngineIsRunningOrDie, containerInfo } from 'playwright-core/lib/containers/docker';
export const dockerPlugin: TestRunnerPlugin = {
name: 'playwright:docker',
async setup(config: FullConfig, configDir: string, rootSuite: Suite, reporter: Reporter) {
async setup(config: FullConfig, configDir: string, reporter: Reporter) {
if (!process.env.PLAYWRIGHT_DOCKER)
return;

3
packages/playwright-test/src/plugins/index.ts
View file

@ -20,7 +20,8 @@ import type { FullConfig } from '../types';
export interface TestRunnerPlugin {
name: string;
setup?(config: FullConfig, configDir: string, rootSuite: Suite, reporter: Reporter): Promise<void>;
setup?(config: FullConfig, configDir: string, reporter: Reporter): Promise<void>;
begin?(suite: Suite): Promise<void>;
teardown?(): Promise<void>;
}

7
packages/playwright-test/src/plugins/vitePlugin.ts
View file

@ -46,11 +46,16 @@ export function createPlugin(
registerSourceFile: string,
frameworkPluginFactory: () => Promise<Plugin>): TestRunnerPlugin {
let configDir: string;
let config: FullConfig;
return {
name: 'playwright-vite-plugin',
setup: async (config: FullConfig, configDirectory: string, suite: Suite) => {
setup: async (configObject: FullConfig, configDirectory: string) => {
config = configObject;
configDir = configDirectory;
},
begin: async (suite: Suite) => {
const use = config.projects[0].use as CtConfig;
const port = use.ctPort || 3100;
const viteConfig: InlineConfig = use.ctViteConfig || {};

4
packages/playwright-test/src/plugins/webServerPlugin.ts
View file

@ -21,7 +21,7 @@ import net from 'net';
import { debug } from 'playwright-core/lib/utilsBundle';
import { raceAgainstTimeout, launchProcess } from 'playwright-core/lib/utils';
import type { FullConfig, Reporter, Suite } from '../../types/testReporter';
import type { FullConfig, Reporter } from '../../types/testReporter';
import type { TestRunnerPlugin } from '.';
import type { FullConfigInternal } from '../types';
import { envWithoutExperimentalLoaderOptions } from '../cli';
@ -57,7 +57,7 @@ export class WebServerPlugin implements TestRunnerPlugin {
this._checkPortOnly = checkPortOnly;
}
public async setup(config: FullConfig, configDir: string, rootSuite: Suite, reporter: Reporter) {
public async setup(config: FullConfig, configDir: string, reporter: Reporter) {
this._reporter = reporter;
this._isAvailable = getIsAvailableFunction(this._options.url, this._checkPortOnly, !!this._options.ignoreHTTPSErrors, this._reporter.onStdErr?.bind(this._reporter));
this._options.cwd = this._options.cwd ? path.resolve(configDir, this._options.cwd) : configDir;

11
packages/playwright-test/src/reporters/base.ts
View file

@ -17,8 +17,8 @@
import { colors, ms as milliseconds, parseStackTraceLine } from 'playwright-core/lib/utilsBundle';
import fs from 'fs';
import path from 'path';
import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location } from '../../types/testReporter';
import type { FullConfigInternal, ReporterInternal } from '../types';
import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location, Reporter } from '../../types/testReporter';
import type { FullConfigInternal } from '../types';
import { codeFrameColumns } from '../babelBundle';
import { monotonicTime } from 'playwright-core/lib/utils';
@ -46,7 +46,7 @@ type TestSummary = {
fatalErrors: TestError[];
};
export class BaseReporter implements ReporterInternal {
export class BaseReporter implements Reporter {
duration = 0;
config!: FullConfigInternal;
suite!: Suite;
@ -63,9 +63,12 @@ export class BaseReporter implements ReporterInternal {
this._ttyWidthForTest = parseInt(process.env.PWTEST_TTY_WIDTH || '', 10);
}
onBegin(config: FullConfig, suite: Suite) {
onConfigure(config: FullConfig) {
this.monotonicStartTime = monotonicTime();
this.config = config as FullConfigInternal;
}
onBegin(config: FullConfig, suite: Suite) {
this.suite = suite;
this.totalTestCount = suite.allTests().length;
}

13
packages/playwright-test/src/reporters/html.ts
View file

@ -20,13 +20,13 @@ import { open } from '../utilsBundle';
import path from 'path';
import type { TransformCallback } from 'stream';
import { Transform } from 'stream';
import type { FullConfig, Suite } from '../../types/testReporter';
import type { FullConfig, Reporter, Suite } from '../../types/testReporter';
import { HttpServer, assert, calculateSha1, monotonicTime, copyFileAndMakeWritable, removeFolders } from 'playwright-core/lib/utils';
import type { JsonAttachment, JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from './raw';
import RawReporter from './raw';
import { stripAnsiEscapes } from './base';
import { getPackageJsonPath, sanitizeForFilePath } from '../util';
import type { FullConfigInternal, Metadata, ReporterInternal } from '../types';
import type { FullConfigInternal, Metadata } from '../types';
import type { ZipFile } from 'playwright-core/lib/zipBundle';
import { yazl } from 'playwright-core/lib/zipBundle';
import { mime } from 'playwright-core/lib/utilsBundle';
@ -47,7 +47,7 @@ type HtmlReporterOptions = {
port?: number,
};
class HtmlReporter implements ReporterInternal {
class HtmlReporter implements Reporter {
private config!: FullConfigInternal;
private suite!: Suite;
private _montonicStartTime: number = 0;
@ -64,9 +64,12 @@ class HtmlReporter implements ReporterInternal {
return false;
}
onBegin(config: FullConfig, suite: Suite) {
onConfigure(config: FullConfig) {
this._montonicStartTime = monotonicTime();
this.config = config as FullConfigInternal;
}
onBegin(config: FullConfig, suite: Suite) {
const { outputFolder, open } = this._resolveOptions();
this._outputFolder = outputFolder;
this._open = open;
@ -112,7 +115,7 @@ class HtmlReporter implements ReporterInternal {
this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports);
}
async _onExit() {
async onExit() {
if (process.env.CI)
return;

5
packages/playwright-test/src/reporters/json.ts
View file

@ -39,8 +39,11 @@ class JSONReporter implements Reporter {
return !this._outputFile;
}
onBegin(config: FullConfig, suite: Suite) {
onConfigure(config: FullConfig) {
this.config = config;
}
onBegin(config: FullConfig, suite: Suite) {
this.suite = suite;
}

5
packages/playwright-test/src/reporters/junit.ts
View file

@ -48,8 +48,11 @@ class JUnitReporter implements Reporter {
return !this.outputFile;
}
onBegin(config: FullConfig, suite: Suite) {
onConfigure(config: FullConfig) {
this.config = config;
}
onBegin(config: FullConfig, suite: Suite) {
this.suite = suite;
this.timestamp = Date.now();
this.startTime = monotonicTime();

18
packages/playwright-test/src/reporters/multiplexer.ts
View file

@ -14,13 +14,12 @@
* limitations under the License.
*/
import type { FullConfig, Suite, TestCase, TestError, TestResult, FullResult, TestStep } from '../../types/testReporter';
import type { ReporterInternal } from '../types';
import type { FullConfig, Suite, TestCase, TestError, TestResult, FullResult, TestStep, Reporter } from '../../types/testReporter';
export class Multiplexer implements ReporterInternal {
private _reporters: ReporterInternal[];
export class Multiplexer implements Reporter {
private _reporters: Reporter[];
constructor(reporters: ReporterInternal[]) {
constructor(reporters: Reporter[]) {
this._reporters = reporters;
}
@ -28,6 +27,11 @@ export class Multiplexer implements ReporterInternal {
return this._reporters.some(r => r.printsToStdio ? r.printsToStdio() : true);
}
onConfigure(config: FullConfig) {
for (const reporter of this._reporters)
reporter.onConfigure?.(config);
}
onBegin(config: FullConfig, suite: Suite) {
for (const reporter of this._reporters)
reporter.onBegin?.(config, suite);
@ -58,9 +62,9 @@ export class Multiplexer implements ReporterInternal {
await Promise.resolve().then(() => reporter.onEnd?.(result)).catch(e => console.error('Error in reporter', e));
}
async _onExit() {
async onExit() {
for (const reporter of this._reporters)
await Promise.resolve().then(() => reporter._onExit?.()).catch(e => console.error('Error in reporter', e));
await Promise.resolve().then(() => reporter.onExit?.()).catch(e => console.error('Error in reporter', e));
}
onError(error: TestError) {

123
packages/playwright-test/src/runner.ts
View file

@ -17,7 +17,7 @@
import * as fs from 'fs';
import * as path from 'path';
import { raceAgainstTimeout } from 'playwright-core/lib/utils';
import { monotonicTime, raceAgainstTimeout } from 'playwright-core/lib/utils';
import { colors, minimatch, rimraf } from 'playwright-core/lib/utilsBundle';
import { promisify } from 'util';
import type { FullResult, Reporter, TestError } from '../types/testReporter';
@ -41,7 +41,7 @@ import { Multiplexer } from './reporters/multiplexer';
import { SigIntWatcher } from './sigIntWatcher';
import type { TestCase } from './test';
import { Suite } from './test';
import type { Config, FullConfigInternal, FullProjectInternal, ReporterInternal } from './types';
import type { Config, FullConfigInternal, FullProjectInternal } from './types';
import { createFileMatcher, createFileMatcherFromFilters, createTitleMatcher, serializeError } from './util';
import type { Matcher, TestFileFilter } from './util';
import { setFatalErrorSink } from './globals';
@ -83,7 +83,7 @@ export type ConfigCLIOverrides = {
export class Runner {
private _configLoader: ConfigLoader;
private _reporter!: ReporterInternal;
private _reporter!: Reporter;
private _plugins: TestRunnerPlugin[] = [];
private _fatalErrors: TestError[] = [];
@ -176,30 +176,6 @@ export class Runner {
return new Multiplexer(reporters);
}
async runAllTests(options: RunOptions): Promise<FullResult> {
this._reporter = await this._createReporter(!!options.listOnly);
const config = this._configLoader.fullConfig();
const result = await raceAgainstTimeout(() => this._run(options), config.globalTimeout);
let fullResult: FullResult;
if (result.timedOut) {
this._reporter.onError?.(createStacklessError(
`Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
fullResult = { status: 'timedout' };
} else {
fullResult = result.result;
}
await this._reporter.onEnd?.(fullResult);
// Calling process.exit() might truncate large stdout/stderr output.
// See https://github.com/nodejs/node/issues/6456.
// See https://github.com/nodejs/node/issues/12921
await new Promise<void>(resolve => process.stdout.write('', () => resolve()));
await new Promise<void>(resolve => process.stderr.write('', () => resolve()));
await this._reporter._onExit?.();
return fullResult;
}
async listTestFiles(projectNames: string[] | undefined): Promise<any> {
const projects = this._collectProjects(projectNames);
const filesByProject = await this._collectFiles(projects, []);
@ -401,7 +377,77 @@ export class Runner {
}
}
private async _run(options: RunOptions): Promise<FullResult> {
async runAllTests(options: RunOptions): Promise<FullResult> {
this._reporter = await this._createReporter(!!options.listOnly);
const config = this._configLoader.fullConfig();
const deadline = config.globalTimeout ? monotonicTime() + config.globalTimeout : 1 << 30;
// Run configure.
this._reporter.onConfigure?.(config);
// Run global setup.
let globalTearDown: (() => Promise<void>) | undefined;
{
const remainingTime = deadline - monotonicTime();
const raceResult = await raceAgainstTimeout(async () => {
const result: FullResult = { status: 'passed' };
globalTearDown = await this._performGlobalSetup(config, result);
return result;
}, remainingTime);
let result: FullResult;
if (raceResult.timedOut) {
this._reporter.onError?.(createStacklessError(
`Timed out waiting ${config.globalTimeout / 1000}s for the global setup to run`));
result = { status: 'timedout' } as FullResult;
} else {
result = raceResult.result;
}
if (result.status !== 'passed')
return result;
}
// Run the tests.
let fullResult: FullResult;
{
const remainingTime = deadline - monotonicTime();
const raceResult = await raceAgainstTimeout(async () => {
try {
return await this._innerRun(options);
} catch (e) {
this._reporter.onError?.(serializeError(e));
return { status: 'failed' } as FullResult;
} finally {
await globalTearDown?.();
}
}, remainingTime);
// If timed out, bail.
let result: FullResult;
if (raceResult.timedOut) {
this._reporter.onError?.(createStacklessError(
`Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
result = { status: 'timedout' };
} else {
result = raceResult.result;
}
// Report end.
await this._reporter.onEnd?.(result);
fullResult = result;
}
// Calling process.exit() might truncate large stdout/stderr output.
// See https://github.com/nodejs/node/issues/6456.
// See https://github.com/nodejs/node/issues/12921
await new Promise<void>(resolve => process.stdout.write('', () => resolve()));
await new Promise<void>(resolve => process.stderr.write('', () => resolve()));
await this._reporter.onExit?.();
return fullResult;
}
private async _innerRun(options: RunOptions): Promise<FullResult> {
const config = this._configLoader.fullConfig();
// Each entry is an array of test groups that can be run concurrently. All
// test groups from the previos entries must finish before entry starts.
@ -415,6 +461,10 @@ export class Runner {
config._maxConcurrentTestGroups = testGroups.length;
const result: FullResult = { status: 'passed' };
for (const plugin of this._plugins)
await plugin.begin?.(rootSuite);
// Report begin
this._reporter.onBegin?.(config, rootSuite);
@ -433,12 +483,6 @@ export class Runner {
if (!await this._removeOutputDirs(options))
return { status: 'failed' };
// Run Global setup.
const result: FullResult = { status: 'passed' };
const globalTearDown = await this._performGlobalSetup(config, rootSuite, result);
if (result.status !== 'passed')
return result;
if (config._ignoreSnapshots) {
this._reporter.onStdOut?.(colors.dim([
'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:',
@ -449,7 +493,6 @@ export class Runner {
}
// Run tests.
try {
const dispatchResult = await this._dispatchToWorkers(testGroups);
if (dispatchResult === 'signal') {
result.status = 'interrupted';
@ -457,12 +500,6 @@ export class Runner {
const failed = dispatchResult === 'workererror' || rootSuite.allTests().some(test => !test.ok());
result.status = failed ? 'failed' : 'passed';
}
} catch (e) {
this._reporter.onError?.(serializeError(e));
return { status: 'failed' };
} finally {
await globalTearDown?.();
}
return result;
}
@ -510,7 +547,7 @@ export class Runner {
return true;
}
private async _performGlobalSetup(config: FullConfigInternal, rootSuite: Suite, result: FullResult): Promise<(() => Promise<void>) | undefined> {
private async _performGlobalSetup(config: FullConfigInternal, result: FullResult): Promise<(() => Promise<void>) | undefined> {
let globalSetupResult: any = undefined;
const pluginsThatWereSetUp: TestRunnerPlugin[] = [];
@ -545,7 +582,7 @@ export class Runner {
// config's global setup.
for (const plugin of this._plugins) {
await Promise.race([
plugin.setup?.(config, config._configDir, rootSuite, this._reporter),
plugin.setup?.(config, config._configDir, this._reporter),
sigintWatcher.promise(),
]);
if (sigintWatcher.hadSignal())

6
packages/playwright-test/src/types.ts
View file

@ -15,7 +15,7 @@
*/
import type { Fixtures, TestInfoError, Project } from '../types/test';
import type { Location, Reporter } from '../types/testReporter';
import type { Location } from '../types/testReporter';
import type { FullConfig as FullConfigPublic, FullProject as FullProjectPublic } from './types';
export * from '../types/test';
export type { Location } from '../types/testReporter';
@ -70,8 +70,4 @@ export interface FullProjectInternal extends FullProjectPublic {
snapshotPathTemplate: string;
}
export interface ReporterInternal extends Reporter {
_onExit?(): void | Promise<void>;
}
export type ContextReuseMode = 'none' | 'force' | 'when-possible';

14
packages/playwright-test/types/testReporter.d.ts vendored
View file

@ -349,6 +349,8 @@ export interface FullResult {
* ```
*
* Here is a typical order of reporter calls:
* - [reporter.onConfigure(config)](https://playwright.dev/docs/api/class-reporter#reporter-on-configure) is called
* once config has been resolved.
* - [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) is called
* once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite].
* - [reporter.onTestBegin(test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-test-begin) is
@ -365,6 +367,8 @@ export interface FullResult {
* [testResult.error](https://playwright.dev/docs/api/class-testresult#test-result-error) and more.
* - [reporter.onEnd(result)](https://playwright.dev/docs/api/class-reporter#reporter-on-end) is called once after
* all tests that should run had finished.
* - [reporter.onExit()](https://playwright.dev/docs/api/class-reporter#reporter-on-exit) is called before test
* runner exits.
*
* Additionally,
* [reporter.onStdOut(chunk, test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-std-out) and
@ -379,6 +383,11 @@ export interface FullResult {
* to enhance user experience.
*/
export interface Reporter {
/**
* Called once config is resolved.
* @param config Resolved configuration.
*/
onConfigure?(config: FullConfig): void;
/**
* Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s.
* @param config Resolved configuration.
@ -397,6 +406,11 @@ export interface Reporter {
* - `'interrupted'` - Interrupted by the user.
*/
onEnd?(result: FullResult): void | Promise<void>;
/**
* Called before test runner exits.
*/
onExit?(): void;
/**
* Called on some global error, for example unhandled exception in the worker process.
* @param error The error.

6
tests/playwright-test/global-setup.spec.ts
View file

@ -137,11 +137,7 @@ test('globalTeardown does not run when globalSetup times out', async ({ runInlin
});
`,
});
// We did not run tests, so we should only have 1 skipped test.
expect(result.skipped).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(0);
expect(result.exitCode).toBe(1);
expect(result.output).toContain('Timed out waiting 1s for the global setup to run');
expect(result.output).not.toContain('teardown=');
});

1
utils/generate_types/overrides-testReporter.d.ts vendored
View file

@ -44,6 +44,7 @@ export interface FullResult {
}
export interface Reporter {
onConfigure?(config: FullConfig): void;
onBegin?(config: FullConfig, suite: Suite): void;
onEnd?(result: FullResult): void | Promise<void>;
}