From ea4eebeb2d1f5984e9dee504aa7501349ad762cc Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 18 Jun 2021 17:56:59 -0700 Subject: [PATCH] feat(test-runner): document tagging, implement grep-invert (#7227) --- docs/src/test-annotations.md | 122 +++++++++++++++++++---- src/test/cli.ts | 2 + src/test/loader.ts | 13 +++ src/test/runner.ts | 6 +- tests/playwright-test/match-grep.spec.ts | 6 ++ types/test.d.ts | 6 ++ 6 files changed, 134 insertions(+), 21 deletions(-) diff --git a/docs/src/test-annotations.md b/docs/src/test-annotations.md index f036c4f7c4..53bf7f1b13 100644 --- a/docs/src/test-annotations.md +++ b/docs/src/test-annotations.md @@ -3,35 +3,77 @@ id: test-annotations title: "Annotations" --- -Sadly, tests do not always pass. Luckily, Playwright Test supports test annotations to deal with failures, flakiness and tests that are not yet ready. + + +## Annotations + +Playwright Test supports test annotations to deal with failures, flakiness, skip, focus and tag tests: +- `skip` marks the test as irrelevant. Playwright Test does not run such a test. Use this annotation when the test is not applicable in some configuration. +- `fail` marks the test as failing. Playwright Test will run this test and ensure it does indeed fail. If the test does not fail, Playwright Test will complain. +- `fixme` marks the test as failing. Playwright Test will not run this test, as opposite to the `fail` annotation. Use `fixme` when running the test is slow or crashy. +- `slow` marks the test as slow and triples the test timeout. + +## Focus a test + +You can focus some tests. When there are focused tests, only these tests run. ```js js-flavor=js -// example.spec.js -const { test, expect } = require('@playwright/test'); - -test('some feature', async ({ page, browserName }) => { - test.skip(browserName !== 'webkit', 'This feature is iOS-only'); - // Test goes here. -}); - -test('another feature', async ({ page }) => { - test.fail(true, 'Broken, need to fix!'); - // Test goes here. +test.only('focus this test', async ({ page }) => { + // Run only focused tests in the entire project. +}); +``` + +```js js-flavor=ts +test.only('focus this test', async ({ page }) => { + // Run only focused tests in the entire project. +}); +``` + +## Skip a test + +You can skip certain tests based on the condition. + +```js js-flavor=js +test('skip this test', async ({ page, browserName }) => { + test.skip(browserName === 'firefox', 'Still working on it'); +}); +``` + +```js js-flavor=ts +test('skip this test', async ({ page, browserName }) => { + test.skip(browserName === 'firefox', 'Still working on it'); +}); +``` + +## Group tests + +You can group tests to give them a logical name or to scope before/after hooks to the group. + +```js js-flavor=js +const { test, expect } = require('@playwright/test'); + +test.describe('two tests', () => { + test('one', async ({ page }) => { + // ... + }); + + test('two', async ({ page }) => { + // ... + }); }); ``` ```js js-flavor=ts -// example.spec.ts import { test, expect } from '@playwright/test'; -test('some feature', async ({ page, browserName }) => { - test.skip(browserName !== 'webkit', 'This feature is iOS-only'); - // Test goes here. -}); +test.describe('two tests', () => { + test('one', async ({ page }) => { + // ... + }); -test('broken feature', async ({ page }) => { - test.fail(); - // Test goes here. + test('two', async ({ page }) => { + // ... + }); }); ``` @@ -42,3 +84,43 @@ Available annotations: - `fail` marks the test as failing. Playwright Test will run this test and ensure it does indeed fail. If the test does not fail, Playwright Test will complain. - `fixme` marks the test as failing. Playwright Test will not run this test, as opposite to the `fail` annotation. Use `fixme` when running the test is slow or crashy. - `slow` marks the test as slow and triples the test timeout. + +## Tag tests + +Sometimes you want to tag your tests as `@fast` or `@slow` and only run the tests that have the certain tag. We recommend that you use the `--grep` and `--grep-invert` command line flags for that: + +```js js-flavor=js +const { test, expect } = require('@playwright/test'); + +test('Test login page @fast', async ({ page }) => { + // ... +}); + +test('Test full report @slow', async ({ page }) => { + // ... +}); +``` + +```js js-flavor=ts +import { test, expect } from '@playwright/test'; + +test('Test login page @fast', async ({ page }) => { + // ... +}); + +test('Test full report @slow', async ({ page }) => { + // ... +}); +``` + +You will then be able to run only that test: + +```bash +npx playwright test --grep @fast +``` + +Or if you want the opposite, you can skip the tests with a certain tag: + +```bash +npx playwright test --grep-invert @slow +``` diff --git a/src/test/cli.ts b/src/test/cli.ts index 007b50a0bb..87f0ff5ed4 100644 --- a/src/test/cli.ts +++ b/src/test/cli.ts @@ -44,6 +44,7 @@ export function addTestCommand(program: commander.CommanderStatic) { command.option('-c, --config ', `Configuration file, or a test directory with optional "${tsConfig}"/"${jsConfig}"`); command.option('--forbid-only', `Fail if test.only is called (default: false)`); command.option('-g, --grep ', `Only run tests matching this regular expression (default: ".*")`); + command.option('-gv, --grep-invert ', `Only run tests that do not match this regular expression`); command.option('--global-timeout ', `Maximum time this test suite can run in milliseconds (default: unlimited)`); command.option('-j, --workers ', `Number of concurrent workers, use 1 to run in a single worker (default: number of CPU cores / 2)`); command.option('--list', `Collect all the tests and report them, but do not run`); @@ -147,6 +148,7 @@ function overridesFromOptions(options: { [key: string]: any }): Config { forbidOnly: options.forbidOnly ? true : undefined, globalTimeout: isDebuggerAttached ? 0 : (options.globalTimeout ? parseInt(options.globalTimeout, 10) : undefined), grep: options.grep ? forceRegExp(options.grep) : undefined, + grepInvert: options.grepInvert ? forceRegExp(options.grepInvert) : undefined, maxFailures: options.x ? 1 : (options.maxFailures ? parseInt(options.maxFailures, 10) : undefined), outputDir: options.output ? path.resolve(process.cwd(), options.output) : undefined, quiet: options.quiet ? options.quiet : undefined, diff --git a/src/test/loader.ts b/src/test/loader.ts index d3e9a67db2..b4f4447424 100644 --- a/src/test/loader.ts +++ b/src/test/loader.ts @@ -96,6 +96,7 @@ export class Loader { this._fullConfig.globalTeardown = takeFirst(this._configOverrides.globalTeardown, this._config.globalTeardown, baseFullConfig.globalTeardown); this._fullConfig.globalTimeout = takeFirst(this._configOverrides.globalTimeout, this._configOverrides.globalTimeout, this._config.globalTimeout, baseFullConfig.globalTimeout); this._fullConfig.grep = takeFirst(this._configOverrides.grep, this._config.grep, baseFullConfig.grep); + this._fullConfig.grepInvert = takeFirst(this._configOverrides.grepInvert, this._config.grepInvert, baseFullConfig.grepInvert); this._fullConfig.maxFailures = takeFirst(this._configOverrides.maxFailures, this._config.maxFailures, baseFullConfig.maxFailures); this._fullConfig.preserveOutput = takeFirst(this._configOverrides.preserveOutput, this._config.preserveOutput, baseFullConfig.preserveOutput); this._fullConfig.reporter = takeFirst(toReporters(this._configOverrides.reporter), toReporters(this._config.reporter), baseFullConfig.reporter); @@ -262,6 +263,17 @@ function validateConfig(config: Config) { } } + if ('grepInvert' in config && config.grepInvert !== undefined) { + if (Array.isArray(config.grepInvert)) { + config.grepInvert.forEach((item, index) => { + if (!isRegExp(item)) + throw new Error(`config.grepInvert[${index}] must be a RegExp`); + }); + } else if (!isRegExp(config.grepInvert)) { + throw new Error(`config.grep must be a RegExp`); + } + } + if ('maxFailures' in config && config.maxFailures !== undefined) { if (typeof config.maxFailures !== 'number' || config.maxFailures < 0) throw new Error(`config.maxFailures must be a non-negative number`); @@ -402,6 +414,7 @@ const baseFullConfig: FullConfig = { globalTeardown: null, globalTimeout: 0, grep: /.*/, + grepInvert: null, maxFailures: 0, preserveOutput: 'always', projects: [], diff --git a/src/test/runner.ts b/src/test/runner.ts index 96b74772df..b1ee1aaa07 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -169,13 +169,17 @@ export class Runner { const outputDirs = new Set(); const grepMatcher = createMatcher(config.grep); + const grepInvertMatcher = config.grepInvert ? createMatcher(config.grepInvert) : null; for (const project of projects) { for (const file of files.get(project)!) { const fileSuite = fileSuites.get(file); if (!fileSuite) continue; for (const spec of fileSuite._allSpecs()) { - if (grepMatcher(spec._testFullTitle(project.config.name))) + const fullTitle = spec._testFullTitle(project.config.name); + if (grepInvertMatcher?.(fullTitle)) + continue; + if (grepMatcher(fullTitle)) project.generateTests(spec); } } diff --git a/tests/playwright-test/match-grep.spec.ts b/tests/playwright-test/match-grep.spec.ts index 1cfbf0d2b4..9ccb70210a 100644 --- a/tests/playwright-test/match-grep.spec.ts +++ b/tests/playwright-test/match-grep.spec.ts @@ -96,3 +96,9 @@ test('should grep by project name', async ({ runInlineTest }) => { expect(result.failed).toBe(0); expect(result.exitCode).toBe(0); }); + +test('should grep invert test name', async ({ runInlineTest }) => { + const result = await runInlineTest(files, { 'grep-invert': 'BB' }); + expect(result.passed).toBe(6); + expect(result.exitCode).toBe(0); +}); diff --git a/types/test.d.ts b/types/test.d.ts index c659e54e7c..7becea4768 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -150,6 +150,11 @@ interface ConfigBase { grep?: RegExp | RegExp[]; /** + * Filter out tests with a title matching one of the patterns. + */ + grepInvert?: RegExp | RegExp[]; + + /** * The maximum number of test failures for this test run. After reaching this number, * testing will stop and exit with an error. Setting to zero (default) disables this behavior. */ @@ -223,6 +228,7 @@ export interface FullConfig { globalTeardown: string | null; globalTimeout: number; grep: RegExp | RegExp[]; + grepInvert: RegExp | RegExp[] | null; maxFailures: number; preserveOutput: PreserveOutput; projects: FullProject[];