From 98fc8b1739f646936b5fb279429e365ad47c4ad0 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Mon, 31 May 2021 20:17:15 -0700 Subject: [PATCH] docs(test runner): update reporters and snapshots docs (#6811) --- docs/src/test-advanced.md | 2 +- docs/src/test-cli.md | 6 +- docs/src/test-fixtures.md | 2 +- docs/src/test-intro.md | 9 ++- docs/src/test-reporters.md | 132 +++++++++++++++++++++++++++++-------- docs/src/test-snapshots.md | 22 +++++-- 6 files changed, 135 insertions(+), 38 deletions(-) diff --git a/docs/src/test-advanced.md b/docs/src/test-advanced.md index 1add875742..cf1f4156d3 100644 --- a/docs/src/test-advanced.md +++ b/docs/src/test-advanced.md @@ -1,6 +1,6 @@ --- id: test-advanced -title: "Advanced Configuration" +title: "Advanced: Configuration" --- diff --git a/docs/src/test-cli.md b/docs/src/test-cli.md index 24650f4897..0f3d5a6124 100644 --- a/docs/src/test-cli.md +++ b/docs/src/test-cli.md @@ -1,6 +1,6 @@ --- id: test-cli -title: "Advanced Command Line" +title: "Advanced: Command Line" --- ```sh @@ -10,7 +10,7 @@ npx playwright test --help Arguments passed to `npx playwright test` are treated as a filter for test files. For example, `npx playwright test my-spec` will only run tests from files with `my-spec` in the name. -All the options are available in the [configuration file](./test-configuration.md). However, selected options can be passed to a command line and take a priority over the configuration file. +All the options are available in the [configuration file](./test-advanced.md). However, selected options can be passed to a command line and take a priority over the configuration file. - `--headed`: Run tests in headed browsers. Useful for debugging. @@ -44,7 +44,7 @@ All the options are available in the [configuration file](./test-configuration.m - `--shard `: [Shard](./test-parallel.md#shards) tests and execute only selected shard, specified in the form `current/all`, 1-based, for example `3/5`. -- `--timeout `: Maximum timeout in milliseconds for each test, defaults to 10 seconds. +- `--timeout `: Maximum timeout in milliseconds for each test, defaults to 30 seconds. - `--update-snapshots` or `-u`: Whether to update [snapshots](./test-snapshots.md) with actual results instead of comparing them. Use this when snapshot expectations have changed. diff --git a/docs/src/test-fixtures.md b/docs/src/test-fixtures.md index f9e11107b9..559315ecfe 100644 --- a/docs/src/test-fixtures.md +++ b/docs/src/test-fixtures.md @@ -1,6 +1,6 @@ --- id: test-fixtures -title: "Advanced Fixtures" +title: "Advanced: Fixtures" --- diff --git a/docs/src/test-intro.md b/docs/src/test-intro.md index 2af7f29b0a..bda31fdd51 100644 --- a/docs/src/test-intro.md +++ b/docs/src/test-intro.md @@ -208,13 +208,14 @@ test.describe('feature foo', () => { ### Write assertions -Playwright Test uses [expect](https://jestjs.io/docs/expect) library for test assertions. It provides a lot of matchers like `toEqual`, `toContain`, `toMatch` and many more. +Playwright Test uses [expect](https://jestjs.io/docs/expect) library for test assertions. It provides a lot of matchers like `toEqual`, `toContain`, `toMatch`, `toMatchSnapshot` and many more. Combine `expect` with various Playwright methods to create expectations for your test: - [`method: Page.isVisible`] - [`method: Page.waitForSelector`] - [`method: Page.textContent`] - [`method: Page.getAttribute`] +- [`method: Page.screenshot`] - Find out more in the [assertions](./assertions.md) guide ```js @@ -236,6 +237,9 @@ test('my test', async ({ page }) => { await page.click('text=Get Started'); // Expect some text to be visible on the page. expect(await page.waitForSelector('text=Getting Started')).toBeTruthy(); + + // Compare screenshot with a stored reference. + expect(await page.screenshot()).toMatchSnapshot('get-started.png'); }); ``` @@ -258,6 +262,9 @@ test('my test', async ({ page }) => { await page.click('text=Get Started'); // Expect some text to be visible on the page. expect(await page.waitForSelector('text=Getting Started')).toBeTruthy(); + + // Compare screenshot with a stored reference. + expect(await page.screenshot()).toMatchSnapshot('get-started.png'); }); ``` diff --git a/docs/src/test-reporters.md b/docs/src/test-reporters.md index d676fa36fa..4bcd9a4358 100644 --- a/docs/src/test-reporters.md +++ b/docs/src/test-reporters.md @@ -14,26 +14,49 @@ Playwright Test comes with a few built-in reporters for different needs and abil npx playwright test --reporter=line ``` -For more control, you can specify reporters programmatically in the [configuration file](#writing-a-configuration-file). +For more control, you can specify reporters programmatically in the [configuration file](./test-configuration.md). ```js -// pwtest.config.ts +// playwright.config.js +module.exports = { + reporter: 'line', +}; +``` + +```ts +// playwright.config.ts import { PlaywrightTestConfig } from 'playwright/test'; const config: PlaywrightTestConfig = { - reporter: 'dot', + reporter: 'line', }; +export default config; +``` -// More complex example: -const config2: PlaywrightTestConfig = { +You can use different reporters locally and on CI. + +```js +// playwright.config.js +module.exports = { reporter: !process.env.CI - // A long list of tests for the terminal. + // A list of tests for the terminal ? 'list' - // Entirely different config on CI. - // Use very concise "dot" reporter plus a comprehensive json report. + // Very concise "dot" reporter and a comprehensive json report for CI : ['dot', { name: 'json', outputFile: 'test-results.json' }], }; +``` +```ts +// playwright.config.ts +import { PlaywrightTestConfig } from 'playwright/test'; + +const config: PlaywrightTestConfig = { + reporter: !process.env.CI + // A list of tests for the terminal + ? 'list' + // Very concise "dot" reporter and a comprehensive json report for CI + : ['dot', { name: 'json', outputFile: 'test-results.json' }], +}; export default config; ``` @@ -43,11 +66,24 @@ All built-in reporters show detailed information about failures, and mostly diff ### List reporter -List reporter is default. It prints a line for each test being run. Use it with `--reporter=list` or `reporter: 'list'`. +List reporter is default. It prints a line for each test being run. + +```sh +npx playwright test --reporter=list +``` ```js -// pwtest.config.ts -const config = { +// playwright.config.js +module.exports = { + reporter: 'list', +}; +``` + +```ts +// playwright.config.ts +import { PlaywrightTestConfig } from 'playwright/test'; + +const config: PlaywrightTestConfig = { reporter: 'list', }; export default config; @@ -72,11 +108,24 @@ Running 124 tests using 6 workers ### Line reporter -Line reporter is more concise than the list reporter. It uses a single line to report last finished test, and prints failures when they occur. Line reporter is useful for large test suites where it shows the progress but does not spam the output by listing all the tests. Use it with `--reporter=line` or `reporter: 'line'`. +Line reporter is more concise than the list reporter. It uses a single line to report last finished test, and prints failures when they occur. Line reporter is useful for large test suites where it shows the progress but does not spam the output by listing all the tests. + +```sh +npx playwright test --reporter=line +``` ```js -// pwtest.config.ts -const config = { +// playwright.config.js +module.exports = { + reporter: 'line', +}; +``` + +```ts +// playwright.config.ts +import { PlaywrightTestConfig } from 'playwright/test'; + +const config: PlaywrightTestConfig = { reporter: 'line', }; export default config; @@ -98,11 +147,24 @@ Running 124 tests using 6 workers ### Dot reporter -Dot reporter is very concise - it only produces a single character per successful test run. It is useful on CI where you don't want a lot of output. Use it with `--reporter=dot` or `reporter: 'dot'`. +Dot reporter is very concise - it only produces a single character per successful test run. It is useful on CI where you don't want a lot of output. + +```sh +npx playwright test --reporter=dot +``` ```js -// pwtest.config.ts -const config = { +// playwright.config.js +module.exports = { + reporter: 'dot', +}; +``` + +```ts +// playwright.config.ts +import { PlaywrightTestConfig } from 'playwright/test'; + +const config: PlaywrightTestConfig = { reporter: 'dot', }; export default config; @@ -119,15 +181,24 @@ Running 124 tests using 6 workers JSON reporter produces an object with all information about the test run. It is usually used together with some terminal reporter like `dot` or `line`. -Most likely you want to write the JSON to a file. When running with `--reporter=json`, use `FOLIO_JSON_OUTPUT_NAME` environment variable: +Most likely you want to write the JSON to a file. When running with `--reporter=json`, use `PLAYWRIGHT_JSON_OUTPUT_NAME` environment variable: ```sh -FOLIO_JSON_OUTPUT_NAME=results.json npx playwright test --reporter=json,dot +PLAYWRIGHT_JSON_OUTPUT_NAME=results.json npx playwright test --reporter=json,dot ``` In configuration file, pass options directly: ```js -// pwtest.config.ts -const config = { +// playwright.config.js +module.exports = { + reporter: { name: 'json', outputFile: 'results.json' }, +}; +``` + +```js +// playwright.config.ts +import { PlaywrightTestConfig } from 'playwright/test'; + +const config: PlaywrightTestConfig = { reporter: { name: 'json', outputFile: 'results.json' }, }; export default config; @@ -137,16 +208,25 @@ export default config; JUnit reporter produces a JUnit-style xml report. It is usually used together with some terminal reporter like `dot` or `line`. -Most likely you want to write the report to an xml file. When running with `--reporter=junit`, use `FOLIO_JUNIT_OUTPUT_NAME` environment variable: +Most likely you want to write the report to an xml file. When running with `--reporter=junit`, use `PLAYWRIGHT_JUNIT_OUTPUT_NAME` environment variable: ```sh -FOLIO_JUNIT_OUTPUT_NAME=results.xml npx playwright test --reporter=junit,line +PLAYWRIGHT_JUNIT_OUTPUT_NAME=results.xml npx playwright test --reporter=junit,line ``` In configuration file, pass options directly: ```js -// pwtest.config.ts -const config = { +// playwright.config.js +module.exports = { + reporter: { name: 'junit', outputFile: 'results.xml' }, +}; +``` + +```js +// playwright.config.ts +import { PlaywrightTestConfig } from 'playwright/test'; + +const config: PlaywrightTestConfig = { reporter: { name: 'junit', outputFile: 'results.xml' }, }; export default config; -``` \ No newline at end of file +``` diff --git a/docs/src/test-snapshots.md b/docs/src/test-snapshots.md index 68904c9f8b..be6edfc5b5 100644 --- a/docs/src/test-snapshots.md +++ b/docs/src/test-snapshots.md @@ -6,13 +6,23 @@ title: "Snapshots" Playwright Test includes the ability to produce and compare snapshots. For that, use `expect(value).toMatchSnapshot()`. Test runner auto-detects the content type, and includes built-in matchers for text, png and jpeg images, and arbitrary binary data. ```js -// example.spec.ts -import { test } from 'playwright/test'; +// example.spec.js +const { test, expect } = require('playwright/test'); -test('my test', async () => { - const image = await produceSomePNG(); - test.expect(image).toMatchSnapshot('optional-snapshot-name.png'); +test('example test', async ({ page }) => { + await page.goto('https://playwright.dev'); + expect(await page.screenshot()).toMatchSnapshot('optional-snapshot-name.png'); }); ``` -Snapshots are stored under `__snapshots__` directory by default, and can be specified in the [configuration object](#configuration-object). +```ts +// example.spec.ts +import { test, expect } from 'playwright/test'; + +test('example test', async ({ page }) => { + await page.goto('https://playwright.dev'); + expect(await page.screenshot()).toMatchSnapshot('optional-snapshot-name.png'); +}); +``` + +Snapshots are stored next to the test file, in a separate directory. For example, `my.spec.js` file will produce and store snapshots in the `my.spec.js-snapshots` directory. You should commit this directory to your version control (e.g. `git`), and review any changes to it.