Merge branch 'main' into sharding-algorithm
This commit is contained in:
Mathias Leppich
commit
e7d07bc797
2
.github/workflows/tests_bidi.yml
vendored
2
.github/workflows/tests_bidi.yml
vendored
|
|
@ -42,3 +42,5 @@ jobs:
|
|||
if: matrix.channel == 'bidi-firefox-beta'
|
||||
- name: Run tests
|
||||
run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run biditest -- --project=${{ matrix.channel }}*
|
||||
env:
|
||||
PWTEST_USE_BIDI_EXPECTATIONS: '1'
|
||||
|
|
|
|||
6
.github/workflows/tests_secondary.yml
vendored
6
.github/workflows/tests_secondary.yml
vendored
|
|
@ -50,7 +50,9 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [macos-12, macos-13, macos-14]
|
||||
# Intel: macos-13, macos-14-large
|
||||
# Arm64: macos-13-xlarge, macos-14
|
||||
os: [macos-13, macos-13-xlarge, macos-14-large, macos-14]
|
||||
browser: [chromium, firefox, webkit]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
|
|
@ -235,7 +237,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-20.04, macos-12, windows-latest]
|
||||
os: [ubuntu-20.04, macos-13, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/run-test
|
||||
|
|
|
|||
|
|
@ -459,9 +459,13 @@ Google Chrome and Microsoft Edge respect enterprise policies, which include limi
|
|||
|
||||
Playwright's Firefox version matches the recent [Firefox Stable](https://www.mozilla.org/en-US/firefox/new/) build. Playwright doesn't work with the branded version of Firefox since it relies on patches.
|
||||
|
||||
Note that availability of certain features, which depend heavily on the underlying platform, may vary between operating systems. For example, available media codecs vary substantially between Linux, macOS and Windows.
|
||||
|
||||
### WebKit
|
||||
|
||||
Playwright's WebKit is derived from the latest WebKit main branch sources, often before these updates are incorporated into Apple Safari and other WebKit-based browsers. This gives a lot of lead time to react on the potential browser update issues. Playwright doesn't work with the branded version of Safari since it relies on patches. Instead, you can test using the most recent WebKit build. Note that availability of certain features, which depend heavily on the underlying platform, may vary between operating systems.
|
||||
Playwright's WebKit is derived from the latest WebKit main branch sources, often before these updates are incorporated into Apple Safari and other WebKit-based browsers. This gives a lot of lead time to react on the potential browser update issues. Playwright doesn't work with the branded version of Safari since it relies on patches. Instead, you can test using the most recent WebKit build.
|
||||
|
||||
Note that availability of certain features, which depend heavily on the underlying platform, may vary between operating systems. For example, available media codecs vary substantially between Linux, macOS and Windows. While running WebKit on Linux CI is usually the most affordable option, for the closest-to-Safari experience you should run WebKit on mac, for example if you do video playback.
|
||||
|
||||
## Install behind a firewall or a proxy
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@
|
|||
},
|
||||
{
|
||||
"name": "webkit",
|
||||
"revision": "2072",
|
||||
"revision": "2073",
|
||||
"installByDefault": true,
|
||||
"revisionOverrides": {
|
||||
"mac10.14": "1446",
|
||||
|
|
|
|||
|
|
@ -31,7 +31,8 @@ export class FailureTracker {
|
|||
}
|
||||
|
||||
onTestEnd(test: TestCase, result: TestResult) {
|
||||
if (result.status !== 'skipped' && result.status !== test.expectedStatus)
|
||||
// Test is considered failing after the last retry.
|
||||
if (test.outcome() === 'unexpected' && test.results.length > test.retries)
|
||||
++this._failureCount;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,12 +15,11 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { monotonicTime } from 'playwright-core/lib/utils';
|
||||
import type { FullResult, TestError } from '../../types/testReporter';
|
||||
import { webServerPluginsForConfig } from '../plugins/webServerPlugin';
|
||||
import { collectFilesForProject, filterProjects } from './projectUtils';
|
||||
import { createErrorCollectingReporter, createReporters } from './reporters';
|
||||
import { TestRun, createTaskRunner, createTaskRunnerForClearCache, createTaskRunnerForDevServer, createTaskRunnerForList, createTaskRunnerForRelatedTestFiles } from './tasks';
|
||||
import { TestRun, createClearCacheTask, createGlobalSetupTasks, createLoadTask, createPluginSetupTasks, createReportBeginTask, createRunTestsTasks, createStartDevServerTask, runTasks } from './tasks';
|
||||
import type { FullConfigInternal } from '../common/config';
|
||||
import { affectedTestFiles } from '../transform/compilationCache';
|
||||
import { InternalReporter } from '../reporters/internalReporter';
|
||||
|
|
@ -69,7 +68,6 @@ export class Runner {
|
|||
async runAllTests(): Promise<FullResult['status']> {
|
||||
const config = this._config;
|
||||
const listOnly = config.cliListOnly;
|
||||
const deadline = config.config.globalTimeout ? monotonicTime() + config.config.globalTimeout : 0;
|
||||
|
||||
// Legacy webServer support.
|
||||
webServerPluginsForConfig(config).forEach(p => config.plugins.push({ factory: p }));
|
||||
|
|
@ -80,24 +78,15 @@ export class Runner {
|
|||
await lastRun.filterLastFailed();
|
||||
|
||||
const reporter = new InternalReporter([...reporters, lastRun]);
|
||||
const taskRunner = listOnly ? createTaskRunnerForList(
|
||||
config,
|
||||
reporter,
|
||||
'in-process',
|
||||
{ failOnLoadErrors: true }) : createTaskRunner(config, reporter);
|
||||
|
||||
const testRun = new TestRun(config);
|
||||
reporter.onConfigure(config.config);
|
||||
|
||||
const taskStatus = await taskRunner.run(testRun, deadline);
|
||||
let status: FullResult['status'] = testRun.failureTracker.result();
|
||||
if (status === 'passed' && taskStatus !== 'passed')
|
||||
status = taskStatus;
|
||||
const modifiedResult = await reporter.onEnd({ status });
|
||||
if (modifiedResult && modifiedResult.status)
|
||||
status = modifiedResult.status;
|
||||
|
||||
await reporter.onExit();
|
||||
const tasks = listOnly ? [
|
||||
createLoadTask('in-process', { failOnLoadErrors: true, filterOnly: false }),
|
||||
createReportBeginTask(),
|
||||
] : [
|
||||
...createGlobalSetupTasks(config),
|
||||
createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }),
|
||||
...createRunTestsTasks(config),
|
||||
];
|
||||
const status = await runTasks(new TestRun(config, reporter), tasks, config.config.globalTimeout);
|
||||
|
||||
// Calling process.exit() might truncate large stdout/stderr output.
|
||||
// See https://github.com/nodejs/node/issues/6456.
|
||||
|
|
@ -110,12 +99,10 @@ export class Runner {
|
|||
async findRelatedTestFiles(files: string[]): Promise<FindRelatedTestFilesReport> {
|
||||
const errorReporter = createErrorCollectingReporter();
|
||||
const reporter = new InternalReporter([errorReporter]);
|
||||
const taskRunner = createTaskRunnerForRelatedTestFiles(this._config, reporter, 'in-process', true);
|
||||
const testRun = new TestRun(this._config);
|
||||
reporter.onConfigure(this._config.config);
|
||||
const status = await taskRunner.run(testRun, 0);
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
const status = await runTasks(new TestRun(this._config, reporter), [
|
||||
...createPluginSetupTasks(this._config),
|
||||
createLoadTask('in-process', { failOnLoadErrors: true, filterOnly: false, populateDependencies: true }),
|
||||
]);
|
||||
if (status !== 'passed')
|
||||
return { errors: errorReporter.errors(), testFiles: [] };
|
||||
return { testFiles: affectedTestFiles(files) };
|
||||
|
|
@ -123,23 +110,21 @@ export class Runner {
|
|||
|
||||
async runDevServer() {
|
||||
const reporter = new InternalReporter([createErrorCollectingReporter(true)]);
|
||||
const taskRunner = createTaskRunnerForDevServer(this._config, reporter, 'in-process', true);
|
||||
const testRun = new TestRun(this._config);
|
||||
reporter.onConfigure(this._config.config);
|
||||
const status = await taskRunner.run(testRun, 0);
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
const status = await runTasks(new TestRun(this._config, reporter), [
|
||||
...createPluginSetupTasks(this._config),
|
||||
createLoadTask('in-process', { failOnLoadErrors: true, filterOnly: false }),
|
||||
createStartDevServerTask(),
|
||||
{ title: 'wait until interrupted', setup: async () => new Promise(() => {}) },
|
||||
]);
|
||||
return { status };
|
||||
}
|
||||
|
||||
async clearCache() {
|
||||
const reporter = new InternalReporter([createErrorCollectingReporter(true)]);
|
||||
const taskRunner = createTaskRunnerForClearCache(this._config, reporter, 'in-process', true);
|
||||
const testRun = new TestRun(this._config);
|
||||
reporter.onConfigure(this._config.config);
|
||||
const status = await taskRunner.run(testRun, 0);
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
const status = await runTasks(new TestRun(this._config, reporter), [
|
||||
...createPluginSetupTasks(this._config),
|
||||
createClearCacheTask(this._config),
|
||||
]);
|
||||
return { status };
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,31 +19,26 @@ import { ManualPromise, monotonicTime } from 'playwright-core/lib/utils';
|
|||
import type { FullResult, TestError } from '../../types/testReporter';
|
||||
import { SigIntWatcher } from './sigIntWatcher';
|
||||
import { serializeError } from '../util';
|
||||
import type { ReporterV2 } from '../reporters/reporterV2';
|
||||
import type { InternalReporter } from '../reporters/internalReporter';
|
||||
|
||||
type TaskPhase<Context> = (reporter: ReporterV2, context: Context, errors: TestError[], softErrors: TestError[]) => Promise<void> | void;
|
||||
export type Task<Context> = { setup?: TaskPhase<Context>, teardown?: TaskPhase<Context> };
|
||||
type TaskPhase<Context> = (context: Context, errors: TestError[], softErrors: TestError[]) => Promise<void> | void;
|
||||
export type Task<Context> = { title: string, setup?: TaskPhase<Context>, teardown?: TaskPhase<Context> };
|
||||
|
||||
export class TaskRunner<Context> {
|
||||
private _tasks: { name: string, task: Task<Context> }[] = [];
|
||||
private _tasks: Task<Context>[] = [];
|
||||
private _reporter: InternalReporter;
|
||||
private _hasErrors = false;
|
||||
private _interrupted = false;
|
||||
private _isTearDown = false;
|
||||
private _globalTimeoutForError: number;
|
||||
|
||||
static create<Context>(reporter: InternalReporter, globalTimeoutForError: number = 0) {
|
||||
return new TaskRunner<Context>(reporter, globalTimeoutForError);
|
||||
}
|
||||
|
||||
private constructor(reporter: InternalReporter, globalTimeoutForError: number) {
|
||||
constructor(reporter: InternalReporter, globalTimeoutForError: number) {
|
||||
this._reporter = reporter;
|
||||
this._globalTimeoutForError = globalTimeoutForError;
|
||||
}
|
||||
|
||||
addTask(name: string, task: Task<Context>) {
|
||||
this._tasks.push({ name, task });
|
||||
addTask(task: Task<Context>) {
|
||||
this._tasks.push(task);
|
||||
}
|
||||
|
||||
async run(context: Context, deadline: number, cancelPromise?: ManualPromise<void>): Promise<FullResult['status']> {
|
||||
|
|
@ -61,18 +56,18 @@ export class TaskRunner<Context> {
|
|||
let currentTaskName: string | undefined;
|
||||
|
||||
const taskLoop = async () => {
|
||||
for (const { name, task } of this._tasks) {
|
||||
currentTaskName = name;
|
||||
for (const task of this._tasks) {
|
||||
currentTaskName = task.title;
|
||||
if (this._interrupted)
|
||||
break;
|
||||
debug('pw:test:task')(`"${name}" started`);
|
||||
debug('pw:test:task')(`"${task.title}" started`);
|
||||
const errors: TestError[] = [];
|
||||
const softErrors: TestError[] = [];
|
||||
try {
|
||||
teardownRunner._tasks.unshift({ name: `teardown for ${name}`, task: { setup: task.teardown } });
|
||||
await task.setup?.(this._reporter, context, errors, softErrors);
|
||||
teardownRunner._tasks.unshift({ title: `teardown for ${task.title}`, setup: task.teardown });
|
||||
await task.setup?.(context, errors, softErrors);
|
||||
} catch (e) {
|
||||
debug('pw:test:task')(`error in "${name}": `, e);
|
||||
debug('pw:test:task')(`error in "${task.title}": `, e);
|
||||
errors.push(serializeError(e));
|
||||
} finally {
|
||||
for (const error of [...softErrors, ...errors])
|
||||
|
|
@ -83,7 +78,7 @@ export class TaskRunner<Context> {
|
|||
this._hasErrors = true;
|
||||
}
|
||||
}
|
||||
debug('pw:test:task')(`"${name}" finished`);
|
||||
debug('pw:test:task')(`"${task.title}" finished`);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import fs from 'fs';
|
|||
import path from 'path';
|
||||
import { promisify } from 'util';
|
||||
import { debug } from 'playwright-core/lib/utilsBundle';
|
||||
import { removeFolders } from 'playwright-core/lib/utils';
|
||||
import { type ManualPromise, monotonicTime, removeFolders } from 'playwright-core/lib/utils';
|
||||
import { Dispatcher, type EnvByProjectId } from './dispatcher';
|
||||
import type { TestRunnerPluginRegistration } from '../plugins';
|
||||
import { createTestGroups, type TestGroup } from '../runner/testGroups';
|
||||
|
|
@ -33,6 +33,7 @@ import { FailureTracker } from './failureTracker';
|
|||
import { detectChangedTestFiles } from './vcs';
|
||||
import type { InternalReporter } from '../reporters/internalReporter';
|
||||
import { cacheDir } from '../transform/compilationCache';
|
||||
import type { FullResult } from '../../types/testReporter';
|
||||
|
||||
const readDirAsync = promisify(fs.readdir);
|
||||
|
||||
|
|
@ -42,132 +43,100 @@ type ProjectWithTestGroups = {
|
|||
testGroups: TestGroup[];
|
||||
};
|
||||
|
||||
export type Phase = {
|
||||
type Phase = {
|
||||
dispatcher: Dispatcher,
|
||||
projects: ProjectWithTestGroups[]
|
||||
};
|
||||
|
||||
export class TestRun {
|
||||
readonly config: FullConfigInternal;
|
||||
readonly reporter: InternalReporter;
|
||||
readonly failureTracker: FailureTracker;
|
||||
rootSuite: Suite | undefined = undefined;
|
||||
readonly phases: Phase[] = [];
|
||||
projectFiles: Map<FullProjectInternal, string[]> = new Map();
|
||||
projectSuites: Map<FullProjectInternal, Suite[]> = new Map();
|
||||
|
||||
constructor(config: FullConfigInternal) {
|
||||
constructor(config: FullConfigInternal, reporter: InternalReporter) {
|
||||
this.config = config;
|
||||
this.reporter = reporter;
|
||||
this.failureTracker = new FailureTracker(config);
|
||||
}
|
||||
}
|
||||
|
||||
export function createTaskRunner(config: FullConfigInternal, reporter: InternalReporter): TaskRunner<TestRun> {
|
||||
const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
|
||||
addGlobalSetupTasks(taskRunner, config);
|
||||
taskRunner.addTask('load tests', createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }));
|
||||
addRunTasks(taskRunner, config);
|
||||
return taskRunner;
|
||||
export async function runTasks(testRun: TestRun, tasks: Task<TestRun>[], globalTimeout?: number, cancelPromise?: ManualPromise<void>) {
|
||||
const deadline = globalTimeout ? monotonicTime() + globalTimeout : 0;
|
||||
const taskRunner = new TaskRunner<TestRun>(testRun.reporter, globalTimeout || 0);
|
||||
for (const task of tasks)
|
||||
taskRunner.addTask(task);
|
||||
testRun.reporter.onConfigure(testRun.config.config);
|
||||
const status = await taskRunner.run(testRun, deadline, cancelPromise);
|
||||
return await finishTaskRun(testRun, status);
|
||||
}
|
||||
|
||||
export function createTaskRunnerForWatchSetup(config: FullConfigInternal, reporter: InternalReporter): TaskRunner<TestRun> {
|
||||
const taskRunner = TaskRunner.create<TestRun>(reporter);
|
||||
addGlobalSetupTasks(taskRunner, config);
|
||||
return taskRunner;
|
||||
export async function runTasksDeferCleanup(testRun: TestRun, tasks: Task<TestRun>[]) {
|
||||
const taskRunner = new TaskRunner<TestRun>(testRun.reporter, 0);
|
||||
for (const task of tasks)
|
||||
taskRunner.addTask(task);
|
||||
testRun.reporter.onConfigure(testRun.config.config);
|
||||
const { status, cleanup } = await taskRunner.runDeferCleanup(testRun, 0);
|
||||
return { status: await finishTaskRun(testRun, status), cleanup };
|
||||
}
|
||||
|
||||
export function createTaskRunnerForTestServer(config: FullConfigInternal, reporter: InternalReporter): TaskRunner<TestRun> {
|
||||
const taskRunner = TaskRunner.create<TestRun>(reporter);
|
||||
taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }));
|
||||
addRunTasks(taskRunner, config);
|
||||
return taskRunner;
|
||||
async function finishTaskRun(testRun: TestRun, status: FullResult['status']) {
|
||||
if (status === 'passed')
|
||||
status = testRun.failureTracker.result();
|
||||
const modifiedResult = await testRun.reporter.onEnd({ status });
|
||||
if (modifiedResult && modifiedResult.status)
|
||||
status = modifiedResult.status;
|
||||
await testRun.reporter.onExit();
|
||||
return status;
|
||||
}
|
||||
|
||||
function addGlobalSetupTasks(taskRunner: TaskRunner<TestRun>, config: FullConfigInternal) {
|
||||
export function createGlobalSetupTasks(config: FullConfigInternal) {
|
||||
const tasks: Task<TestRun>[] = [];
|
||||
if (!config.configCLIOverrides.preserveOutputDir && !process.env.PW_TEST_NO_REMOVE_OUTPUT_DIRS)
|
||||
taskRunner.addTask('clear output', createRemoveOutputDirsTask());
|
||||
for (const plugin of config.plugins)
|
||||
taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
|
||||
tasks.push(createRemoveOutputDirsTask());
|
||||
tasks.push(...createPluginSetupTasks(config));
|
||||
if (config.config.globalSetup || config.config.globalTeardown)
|
||||
taskRunner.addTask('global setup', createGlobalSetupTask());
|
||||
tasks.push(createGlobalSetupTask());
|
||||
return tasks;
|
||||
}
|
||||
|
||||
function addRunTasks(taskRunner: TaskRunner<TestRun>, config: FullConfigInternal) {
|
||||
taskRunner.addTask('create phases', createPhasesTask());
|
||||
taskRunner.addTask('report begin', createReportBeginTask());
|
||||
for (const plugin of config.plugins)
|
||||
taskRunner.addTask('plugin begin', createPluginBeginTask(plugin));
|
||||
taskRunner.addTask('test suite', createRunTestsTask());
|
||||
return taskRunner;
|
||||
export function createRunTestsTasks(config: FullConfigInternal) {
|
||||
return [
|
||||
createPhasesTask(),
|
||||
createReportBeginTask(),
|
||||
...config.plugins.map(plugin => createPluginBeginTask(plugin)),
|
||||
createRunTestsTask(),
|
||||
];
|
||||
}
|
||||
|
||||
export function createTaskRunnerForList(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', options: { failOnLoadErrors: boolean }): TaskRunner<TestRun> {
|
||||
const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
|
||||
taskRunner.addTask('load tests', createLoadTask(mode, { ...options, filterOnly: false }));
|
||||
taskRunner.addTask('report begin', createReportBeginTask());
|
||||
return taskRunner;
|
||||
}
|
||||
|
||||
export function createTaskRunnerForListFiles(config: FullConfigInternal, reporter: InternalReporter): TaskRunner<TestRun> {
|
||||
const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
|
||||
taskRunner.addTask('load tests', createListFilesTask());
|
||||
taskRunner.addTask('report begin', createReportBeginTask());
|
||||
return taskRunner;
|
||||
}
|
||||
|
||||
export function createTaskRunnerForDevServer(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', setupAndWait: boolean): TaskRunner<TestRun> {
|
||||
const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
|
||||
if (setupAndWait) {
|
||||
for (const plugin of config.plugins)
|
||||
taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
|
||||
}
|
||||
taskRunner.addTask('load tests', createLoadTask(mode, { failOnLoadErrors: true, filterOnly: false }));
|
||||
taskRunner.addTask('start dev server', createStartDevServerTask());
|
||||
if (setupAndWait) {
|
||||
taskRunner.addTask('wait until interrupted', {
|
||||
setup: async () => new Promise(() => {}),
|
||||
});
|
||||
}
|
||||
return taskRunner;
|
||||
}
|
||||
|
||||
export function createTaskRunnerForRelatedTestFiles(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', setupPlugins: boolean): TaskRunner<TestRun> {
|
||||
const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
|
||||
if (setupPlugins) {
|
||||
for (const plugin of config.plugins)
|
||||
taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
|
||||
}
|
||||
taskRunner.addTask('load tests', createLoadTask(mode, { failOnLoadErrors: true, filterOnly: false, populateDependencies: true }));
|
||||
return taskRunner;
|
||||
}
|
||||
|
||||
export function createTaskRunnerForClearCache(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', setupPlugins: boolean): TaskRunner<TestRun> {
|
||||
const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
|
||||
if (setupPlugins) {
|
||||
for (const plugin of config.plugins)
|
||||
taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
|
||||
}
|
||||
taskRunner.addTask('clear cache', {
|
||||
export function createClearCacheTask(config: FullConfigInternal): Task<TestRun> {
|
||||
return {
|
||||
title: 'clear cache',
|
||||
setup: async () => {
|
||||
await removeDirAndLogToConsole(cacheDir);
|
||||
for (const plugin of config.plugins)
|
||||
await plugin.instance?.clearCache?.();
|
||||
},
|
||||
});
|
||||
return taskRunner;
|
||||
};
|
||||
}
|
||||
|
||||
function createReportBeginTask(): Task<TestRun> {
|
||||
export function createReportBeginTask(): Task<TestRun> {
|
||||
return {
|
||||
setup: async (reporter, { rootSuite }) => {
|
||||
reporter.onBegin?.(rootSuite!);
|
||||
title: 'report begin',
|
||||
setup: async testRun => {
|
||||
testRun.reporter.onBegin?.(testRun.rootSuite!);
|
||||
},
|
||||
teardown: async ({}) => {},
|
||||
};
|
||||
}
|
||||
|
||||
function createPluginSetupTask(plugin: TestRunnerPluginRegistration): Task<TestRun> {
|
||||
return {
|
||||
setup: async (reporter, { config }) => {
|
||||
export function createPluginSetupTasks(config: FullConfigInternal): Task<TestRun>[] {
|
||||
return config.plugins.map(plugin => ({
|
||||
title: 'plugin setup',
|
||||
setup: async ({ reporter }) => {
|
||||
if (typeof plugin.factory === 'function')
|
||||
plugin.instance = await plugin.factory();
|
||||
else
|
||||
|
|
@ -177,13 +146,14 @@ function createPluginSetupTask(plugin: TestRunnerPluginRegistration): Task<TestR
|
|||
teardown: async () => {
|
||||
await plugin.instance?.teardown?.();
|
||||
},
|
||||
};
|
||||
}));
|
||||
}
|
||||
|
||||
function createPluginBeginTask(plugin: TestRunnerPluginRegistration): Task<TestRun> {
|
||||
return {
|
||||
setup: async (reporter, { rootSuite }) => {
|
||||
await plugin.instance?.begin?.(rootSuite!);
|
||||
title: 'plugin begin',
|
||||
setup: async testRun => {
|
||||
await plugin.instance?.begin?.(testRun.rootSuite!);
|
||||
},
|
||||
teardown: async () => {
|
||||
await plugin.instance?.end?.();
|
||||
|
|
@ -196,13 +166,14 @@ function createGlobalSetupTask(): Task<TestRun> {
|
|||
let globalSetupFinished = false;
|
||||
let teardownHook: any;
|
||||
return {
|
||||
setup: async (reporter, { config }) => {
|
||||
title: 'global setup',
|
||||
setup: async ({ config }) => {
|
||||
const setupHook = config.config.globalSetup ? await loadGlobalHook(config, config.config.globalSetup) : undefined;
|
||||
teardownHook = config.config.globalTeardown ? await loadGlobalHook(config, config.config.globalTeardown) : undefined;
|
||||
globalSetupResult = setupHook ? await setupHook(config.config) : undefined;
|
||||
globalSetupFinished = true;
|
||||
},
|
||||
teardown: async (reporter, { config }) => {
|
||||
teardown: async ({ config }) => {
|
||||
if (typeof globalSetupResult === 'function')
|
||||
await globalSetupResult();
|
||||
if (globalSetupFinished)
|
||||
|
|
@ -213,7 +184,8 @@ function createGlobalSetupTask(): Task<TestRun> {
|
|||
|
||||
function createRemoveOutputDirsTask(): Task<TestRun> {
|
||||
return {
|
||||
setup: async (reporter, { config }) => {
|
||||
title: 'clear output',
|
||||
setup: async ({ config }) => {
|
||||
const outputDirs = new Set<string>();
|
||||
const projects = filterProjects(config.projects, config.cliProjectFilter);
|
||||
projects.forEach(p => outputDirs.add(p.project.outputDir));
|
||||
|
|
@ -235,9 +207,10 @@ function createRemoveOutputDirsTask(): Task<TestRun> {
|
|||
};
|
||||
}
|
||||
|
||||
function createListFilesTask(): Task<TestRun> {
|
||||
export function createListFilesTask(): Task<TestRun> {
|
||||
return {
|
||||
setup: async (reporter, testRun, errors) => {
|
||||
title: 'load tests',
|
||||
setup: async (testRun, errors) => {
|
||||
testRun.rootSuite = await createRootSuite(testRun, errors, false);
|
||||
testRun.failureTracker.onRootSuite(testRun.rootSuite);
|
||||
await collectProjectsAndTestFiles(testRun, false);
|
||||
|
|
@ -258,9 +231,10 @@ function createListFilesTask(): Task<TestRun> {
|
|||
};
|
||||
}
|
||||
|
||||
function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, populateDependencies?: boolean }): Task<TestRun> {
|
||||
export function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, populateDependencies?: boolean }): Task<TestRun> {
|
||||
return {
|
||||
setup: async (reporter, testRun, errors, softErrors) => {
|
||||
title: 'load tests',
|
||||
setup: async (testRun, errors, softErrors) => {
|
||||
await collectProjectsAndTestFiles(testRun, !!options.doNotRunDepsOutsideProjectFilter);
|
||||
await loadFileSuites(testRun, mode, options.failOnLoadErrors ? errors : softErrors);
|
||||
|
||||
|
|
@ -294,7 +268,8 @@ function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filter
|
|||
|
||||
function createPhasesTask(): Task<TestRun> {
|
||||
return {
|
||||
setup: async (reporter, testRun) => {
|
||||
title: 'create phases',
|
||||
setup: async testRun => {
|
||||
let maxConcurrentTestGroups = 0;
|
||||
|
||||
const processed = new Set<FullProjectInternal>();
|
||||
|
|
@ -325,7 +300,7 @@ function createPhasesTask(): Task<TestRun> {
|
|||
processed.add(project);
|
||||
if (phaseProjects.length) {
|
||||
let testGroupsInPhase = 0;
|
||||
const phase: Phase = { dispatcher: new Dispatcher(testRun.config, reporter, testRun.failureTracker), projects: [] };
|
||||
const phase: Phase = { dispatcher: new Dispatcher(testRun.config, testRun.reporter, testRun.failureTracker), projects: [] };
|
||||
testRun.phases.push(phase);
|
||||
for (const project of phaseProjects) {
|
||||
const projectSuite = projectToSuite.get(project)!;
|
||||
|
|
@ -345,7 +320,8 @@ function createPhasesTask(): Task<TestRun> {
|
|||
|
||||
function createRunTestsTask(): Task<TestRun> {
|
||||
return {
|
||||
setup: async (reporter, { phases, failureTracker }) => {
|
||||
title: 'test suite',
|
||||
setup: async ({ phases, failureTracker }) => {
|
||||
const successfulProjects = new Set<FullProjectInternal>();
|
||||
const extraEnvByProjectId: EnvByProjectId = new Map();
|
||||
const teardownToSetups = buildTeardownToSetupsMap(phases.map(phase => phase.projects.map(p => p.project)).flat());
|
||||
|
|
@ -389,28 +365,29 @@ function createRunTestsTask(): Task<TestRun> {
|
|||
}
|
||||
}
|
||||
},
|
||||
teardown: async (reporter, { phases }) => {
|
||||
teardown: async ({ phases }) => {
|
||||
for (const { dispatcher } of phases.reverse())
|
||||
await dispatcher.stop();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createStartDevServerTask(): Task<TestRun> {
|
||||
export function createStartDevServerTask(): Task<TestRun> {
|
||||
return {
|
||||
setup: async (reporter, testRun, errors, softErrors) => {
|
||||
if (testRun.config.plugins.some(plugin => !!plugin.devServerCleanup)) {
|
||||
title: 'start dev server',
|
||||
setup: async ({ config }, errors, softErrors) => {
|
||||
if (config.plugins.some(plugin => !!plugin.devServerCleanup)) {
|
||||
errors.push({ message: `DevServer is already running` });
|
||||
return;
|
||||
}
|
||||
for (const plugin of testRun.config.plugins)
|
||||
for (const plugin of config.plugins)
|
||||
plugin.devServerCleanup = await plugin.instance?.startDevServer?.();
|
||||
if (!testRun.config.plugins.some(plugin => !!plugin.devServerCleanup))
|
||||
if (!config.plugins.some(plugin => !!plugin.devServerCleanup))
|
||||
errors.push({ message: `DevServer is not available in the package you are using. Did you mean to use component testing?` });
|
||||
},
|
||||
|
||||
teardown: async (reporter, testRun) => {
|
||||
for (const plugin of testRun.config.plugins) {
|
||||
teardown: async ({ config }) => {
|
||||
for (const plugin of config.plugins) {
|
||||
await plugin.devServerCleanup?.();
|
||||
plugin.devServerCleanup = undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import type * as reporterTypes from '../../types/testReporter';
|
|||
import { affectedTestFiles, collectAffectedTestFiles, dependenciesForTestFile } from '../transform/compilationCache';
|
||||
import type { ConfigLocation, FullConfigInternal } from '../common/config';
|
||||
import { createErrorCollectingReporter, createReporterForTestServer, createReporters } from './reporters';
|
||||
import { TestRun, createTaskRunnerForList, createTaskRunnerForTestServer, createTaskRunnerForWatchSetup, createTaskRunnerForListFiles, createTaskRunnerForDevServer, createTaskRunnerForRelatedTestFiles, createTaskRunnerForClearCache } from './tasks';
|
||||
import { TestRun, runTasks, createLoadTask, createRunTestsTasks, createReportBeginTask, createListFilesTask, runTasksDeferCleanup, createClearCacheTask, createGlobalSetupTasks, createStartDevServerTask } from './tasks';
|
||||
import { open } from 'playwright-core/lib/utilsBundle';
|
||||
import ListReporter from '../reporters/list';
|
||||
import { SigIntWatcher } from './sigIntWatcher';
|
||||
|
|
@ -150,17 +150,13 @@ export class TestServerDispatcher implements TestServerInterface {
|
|||
if (!config)
|
||||
return { status: 'failed', report };
|
||||
|
||||
const taskRunner = createTaskRunnerForWatchSetup(config, reporter);
|
||||
reporter.onConfigure(config.config);
|
||||
const testRun = new TestRun(config);
|
||||
const { status, cleanup: globalCleanup } = await taskRunner.runDeferCleanup(testRun, 0);
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
if (status !== 'passed') {
|
||||
await globalCleanup();
|
||||
return { report, status };
|
||||
}
|
||||
this._globalSetup = { cleanup: globalCleanup, report };
|
||||
const { status, cleanup } = await runTasksDeferCleanup(new TestRun(config, reporter), [
|
||||
...createGlobalSetupTasks(config),
|
||||
]);
|
||||
if (status !== 'passed')
|
||||
await cleanup();
|
||||
else
|
||||
this._globalSetup = { cleanup, report };
|
||||
return { report, status };
|
||||
}
|
||||
|
||||
|
|
@ -179,17 +175,14 @@ export class TestServerDispatcher implements TestServerInterface {
|
|||
if (!config)
|
||||
return { report, status: 'failed' };
|
||||
|
||||
const taskRunner = createTaskRunnerForDevServer(config, reporter, 'out-of-process', false);
|
||||
const testRun = new TestRun(config);
|
||||
reporter.onConfigure(config.config);
|
||||
const { status, cleanup } = await taskRunner.runDeferCleanup(testRun, 0);
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
if (status !== 'passed') {
|
||||
const { status, cleanup } = await runTasksDeferCleanup(new TestRun(config, reporter), [
|
||||
createLoadTask('out-of-process', { failOnLoadErrors: true, filterOnly: false }),
|
||||
createStartDevServerTask(),
|
||||
]);
|
||||
if (status !== 'passed')
|
||||
await cleanup();
|
||||
return { report, status };
|
||||
}
|
||||
this._devServer = { cleanup, report };
|
||||
else
|
||||
this._devServer = { cleanup, report };
|
||||
return { report, status };
|
||||
}
|
||||
|
||||
|
|
@ -205,13 +198,9 @@ export class TestServerDispatcher implements TestServerInterface {
|
|||
const config = await this._loadConfigOrReportError(reporter);
|
||||
if (!config)
|
||||
return;
|
||||
|
||||
const taskRunner = createTaskRunnerForClearCache(config, reporter, 'out-of-process', false);
|
||||
const testRun = new TestRun(config);
|
||||
reporter.onConfigure(config.config);
|
||||
const status = await taskRunner.run(testRun, 0);
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
await runTasks(new TestRun(config, reporter), [
|
||||
createClearCacheTask(config),
|
||||
]);
|
||||
}
|
||||
|
||||
async listFiles(params: Parameters<TestServerInterface['listFiles']>[0]): ReturnType<TestServerInterface['listFiles']> {
|
||||
|
|
@ -221,12 +210,10 @@ export class TestServerDispatcher implements TestServerInterface {
|
|||
return { status: 'failed', report };
|
||||
|
||||
config.cliProjectFilter = params.projects?.length ? params.projects : undefined;
|
||||
const taskRunner = createTaskRunnerForListFiles(config, reporter);
|
||||
reporter.onConfigure(config.config);
|
||||
const testRun = new TestRun(config);
|
||||
const status = await taskRunner.run(testRun, 0);
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
const status = await runTasks(new TestRun(config, reporter), [
|
||||
createListFilesTask(),
|
||||
createReportBeginTask(),
|
||||
]);
|
||||
return { report, status };
|
||||
}
|
||||
|
||||
|
|
@ -264,12 +251,10 @@ export class TestServerDispatcher implements TestServerInterface {
|
|||
config.cliProjectFilter = params.projects?.length ? params.projects : undefined;
|
||||
config.cliListOnly = true;
|
||||
|
||||
const taskRunner = createTaskRunnerForList(config, reporter, 'out-of-process', { failOnLoadErrors: false });
|
||||
const testRun = new TestRun(config);
|
||||
reporter.onConfigure(config.config);
|
||||
const status = await taskRunner.run(testRun, 0);
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
const status = await runTasks(new TestRun(config, reporter), [
|
||||
createLoadTask('out-of-process', { failOnLoadErrors: false, filterOnly: false }),
|
||||
createReportBeginTask(),
|
||||
]);
|
||||
return { config, report, reporter, status };
|
||||
}
|
||||
|
||||
|
|
@ -344,13 +329,12 @@ export class TestServerDispatcher implements TestServerInterface {
|
|||
|
||||
const configReporters = await createReporters(config, 'test', true);
|
||||
const reporter = new InternalReporter([...configReporters, wireReporter]);
|
||||
const taskRunner = createTaskRunnerForTestServer(config, reporter);
|
||||
const testRun = new TestRun(config);
|
||||
reporter.onConfigure(config.config);
|
||||
const stop = new ManualPromise();
|
||||
const run = taskRunner.run(testRun, 0, stop).then(async status => {
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
const tasks = [
|
||||
createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }),
|
||||
...createRunTestsTasks(config),
|
||||
];
|
||||
const run = runTasks(new TestRun(config, reporter), tasks, 0, stop).then(async status => {
|
||||
this._testRun = undefined;
|
||||
return status;
|
||||
});
|
||||
|
|
@ -373,13 +357,9 @@ export class TestServerDispatcher implements TestServerInterface {
|
|||
const config = await this._loadConfigOrReportError(reporter);
|
||||
if (!config)
|
||||
return { errors: errorReporter.errors(), testFiles: [] };
|
||||
|
||||
const taskRunner = createTaskRunnerForRelatedTestFiles(config, reporter, 'out-of-process', false);
|
||||
const testRun = new TestRun(config);
|
||||
reporter.onConfigure(config.config);
|
||||
const status = await taskRunner.run(testRun, 0);
|
||||
await reporter.onEnd({ status });
|
||||
await reporter.onExit();
|
||||
const status = await runTasks(new TestRun(config, reporter), [
|
||||
createLoadTask('out-of-process', { failOnLoadErrors: true, filterOnly: false, populateDependencies: true }),
|
||||
]);
|
||||
if (status !== 'passed')
|
||||
return { errors: errorReporter.errors(), testFiles: [] };
|
||||
return { testFiles: affectedTestFiles(params.files) };
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ import type {
|
|||
FullConfig, FullResult, Reporter, Suite, TestCase
|
||||
} from '@playwright/test/reporter';
|
||||
import fs from 'fs';
|
||||
import { projectExpectationPath } from './expectationUtil';
|
||||
import { parseBidiExpectations as parseExpectations, projectExpectationPath } from './expectationUtil';
|
||||
import type { TestExpectation } from './expectationUtil';
|
||||
|
||||
type ReporterOptions = {
|
||||
rebase?: boolean;
|
||||
|
|
@ -27,6 +28,7 @@ type ReporterOptions = {
|
|||
class ExpectationReporter implements Reporter {
|
||||
private _suite: Suite;
|
||||
private _options: ReporterOptions;
|
||||
private _pendingUpdates: Promise<void>[] = [];
|
||||
|
||||
constructor(options: ReporterOptions) {
|
||||
this._options = options;
|
||||
|
|
@ -40,18 +42,28 @@ class ExpectationReporter implements Reporter {
|
|||
if (!this._options.rebase)
|
||||
return;
|
||||
for (const project of this._suite.suites)
|
||||
this._updateProjectExpectations(project);
|
||||
this._pendingUpdates.push(this._updateProjectExpectations(project));
|
||||
}
|
||||
|
||||
private _updateProjectExpectations(project: Suite) {
|
||||
const results = project.allTests().map(test => {
|
||||
const outcome = getOutcome(test);
|
||||
const line = `${test.titlePath().slice(1).join(' › ')} [${outcome}]`;
|
||||
return line;
|
||||
});
|
||||
async onExit() {
|
||||
await Promise.all(this._pendingUpdates);
|
||||
}
|
||||
|
||||
private async _updateProjectExpectations(project: Suite) {
|
||||
const outputFile = projectExpectationPath(project.title);
|
||||
const expectations = await parseExpectations(project.title);
|
||||
for (const test of project.allTests()) {
|
||||
const outcome = getOutcome(test);
|
||||
// Strip root and project names.
|
||||
const key = test.titlePath().slice(2).join(' › ');
|
||||
if (!expectations.has(key) || expectations.get(key) === 'unknown')
|
||||
expectations.set(key, outcome);
|
||||
}
|
||||
const keys = Array.from(expectations.keys());
|
||||
keys.sort();
|
||||
const results = keys.map(key => `${key} [${expectations.get(key)}]`);
|
||||
console.log('Writing new expectations to', outputFile);
|
||||
fs.writeFileSync(outputFile, results.join('\n'));
|
||||
await fs.promises.writeFile(outputFile, results.join('\n'));
|
||||
}
|
||||
|
||||
printsToStdio(): boolean {
|
||||
|
|
@ -59,7 +71,7 @@ class ExpectationReporter implements Reporter {
|
|||
}
|
||||
}
|
||||
|
||||
function getOutcome(test: TestCase): 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout' {
|
||||
function getOutcome(test: TestCase): TestExpectation {
|
||||
if (test.results.length === 0)
|
||||
return 'unknown';
|
||||
if (test.results.every(r => r.status === 'timedOut'))
|
||||
|
|
|
|||
|
|
@ -18,14 +18,27 @@ import fs from 'fs';
|
|||
import path from 'path';
|
||||
import type { TestInfo } from 'playwright/test';
|
||||
|
||||
export type TestExpectation = 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout';
|
||||
|
||||
type ShouldSkipPredicate = (info: TestInfo) => boolean;
|
||||
|
||||
export async function parseBidiExpectations(projectName: string): Promise<ShouldSkipPredicate> {
|
||||
export async function createSkipTestPredicate(projectName: string): Promise<ShouldSkipPredicate> {
|
||||
if (!process.env.PWTEST_USE_BIDI_EXPECTATIONS)
|
||||
return () => false;
|
||||
const expectationsMap = await parseBidiExpectations(projectName);
|
||||
return (info: TestInfo) => {
|
||||
const key = info.titlePath.join(' › ');
|
||||
const expectation = expectationsMap.get(key);
|
||||
return expectation === 'fail' || expectation === 'timeout';
|
||||
};
|
||||
}
|
||||
|
||||
export async function parseBidiExpectations(projectName: string): Promise<Map<string, TestExpectation>> {
|
||||
const filePath = projectExpectationPath(projectName);
|
||||
try {
|
||||
await fs.promises.access(filePath);
|
||||
} catch (e) {
|
||||
return () => false;
|
||||
return new Map();
|
||||
}
|
||||
const content = await fs.promises.readFile(filePath);
|
||||
const pairs = content.toString().split('\n').map(line => {
|
||||
|
|
@ -35,14 +48,8 @@ export async function parseBidiExpectations(projectName: string): Promise<Should
|
|||
return undefined;
|
||||
}
|
||||
return [match.groups!.titlePath, match.groups!.expectation];
|
||||
}).filter(Boolean) as [string, string][];
|
||||
const expectationsMap = new Map(pairs);
|
||||
|
||||
return (info: TestInfo) => {
|
||||
const key = [info.project.name, ...info.titlePath].join(' › ');
|
||||
const expectation = expectationsMap.get(key);
|
||||
return expectation === 'fail' || expectation === 'timeout';
|
||||
};
|
||||
}).filter(Boolean) as [string, TestExpectation][];
|
||||
return new Map(pairs);
|
||||
}
|
||||
|
||||
export function projectExpectationPath(project: string): string {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
1911
tests/bidi/expectations/bidi-firefox-beta-library.txt
Normal file
1911
tests/bidi/expectations/bidi-firefox-beta-library.txt
Normal file
File diff suppressed because it is too large
Load diff
1971
tests/bidi/expectations/bidi-firefox-beta-page.txt
Normal file
1971
tests/bidi/expectations/bidi-firefox-beta-page.txt
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -24,7 +24,7 @@ import { baseTest } from './baseTest';
|
|||
import { type RemoteServerOptions, type PlaywrightServer, RunServer, RemoteServer } from './remoteServer';
|
||||
import type { Log } from '../../packages/trace/src/har';
|
||||
import { parseHar } from '../config/utils';
|
||||
import { parseBidiExpectations as parseBidiProjectExpectations } from '../bidi/expectationUtil';
|
||||
import { createSkipTestPredicate } from '../bidi/expectationUtil';
|
||||
import type { TestInfo } from '@playwright/test';
|
||||
|
||||
export type BrowserTestWorkerFixtures = PageWorkerFixtures & {
|
||||
|
|
@ -172,7 +172,7 @@ const test = baseTest.extend<BrowserTestTestFixtures, BrowserTestWorkerFixtures>
|
|||
},
|
||||
|
||||
bidiTestSkipPredicate: [async ({ }, run) => {
|
||||
const filter = await parseBidiProjectExpectations(test.info().project.name);
|
||||
const filter = await createSkipTestPredicate(test.info().project.name);
|
||||
await run(filter);
|
||||
}, { scope: 'worker' }],
|
||||
|
||||
|
|
|
|||
|
|
@ -76,8 +76,8 @@ test('max-failures should work with retries', async ({ runInlineTest }) => {
|
|||
`,
|
||||
}, { 'max-failures': 2, 'retries': 4 });
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.failed).toBe(1);
|
||||
expect(result.output.split('\n').filter(l => l.includes('Received:')).length).toBe(2);
|
||||
expect(result.failed).toBe(2);
|
||||
expect(result.output.split('\n').filter(l => l.includes('Received:')).length).toBe(2 * (4 + 1)); // 2 tests * (4 retries + 1 original)
|
||||
});
|
||||
|
||||
test('max-failures should stop workers', async ({ runInlineTest }) => {
|
||||
|
|
@ -181,3 +181,31 @@ test('max-failures should work across phases', async ({ runInlineTest }) => {
|
|||
expect(result.output).toContain('running c');
|
||||
expect(result.output).not.toContain('running d');
|
||||
});
|
||||
|
||||
test('max-failures should not consider retries as failures', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'playwright.config.ts': `
|
||||
export default {
|
||||
maxFailures: 10,
|
||||
retries: 10,
|
||||
};
|
||||
`,
|
||||
'example.spec.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
|
||||
test('I fail 9 times 1', () => {
|
||||
if (test.info().retry < 9)
|
||||
throw new Error('failing intentionally');
|
||||
});
|
||||
|
||||
test('I fail 9 times 2', () => {
|
||||
if (test.info().retry < 9)
|
||||
throw new Error('failing intentionally');
|
||||
});
|
||||
`,
|
||||
}, { workers: 1 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.flaky).toBe(2);
|
||||
expect(result.passed).toBe(0);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue