From 4578d5790132aae19294e88fc4a3d6c5966cec3e Mon Sep 17 00:00:00 2001
From: Pavel Feldman <pavel.feldman@gmail.com>
Date: Thu, 27 May 2021 09:16:50 -0700
Subject: [PATCH] docs(runner): start adding runner docs (#6773)

---
 docs/src/test-runner-configuration-js.md | 181 +++++++++++++++++++++
 docs/src/test-runner-examples-js.md      | 133 +++++++++++++++
 docs/src/test-runner-intro-js.md         | 198 +++++++++++++++++++++++
 docs/src/test-runners-js.md              |   8 +-
 4 files changed, 518 insertions(+), 2 deletions(-)
 create mode 100644 docs/src/test-runner-configuration-js.md
 create mode 100644 docs/src/test-runner-examples-js.md
 create mode 100644 docs/src/test-runner-intro-js.md

diff --git a/docs/src/test-runner-configuration-js.md b/docs/src/test-runner-configuration-js.md
new file mode 100644
index 0000000000..3a09fae2ee
--- /dev/null
+++ b/docs/src/test-runner-configuration-js.md
@@ -0,0 +1,181 @@
+---
+id: test-runner-configuration
+title: "Configuration"
+---
+
+<!-- TOC -->
+
+## Modify options
+
+You can modify browser launch options, context creation options and testing options either globally in the configuration file, or locally in the test file.
+
+Playwright test runner is based on the [Folio] framework, so it supports any configuration available in Folio, and adds a lot of Playwright-specific options.
+
+### Globally in the configuration file
+
+You can specify different options for each browser using projects in the configuration file. Below is an example that changes some global testing options, and Chromium browser configuration.
+
+```js
+// config.ts
+import { PlaywrightTestConfig } from "@playwright/test";
+
+const config: PlaywrightTestConfig = {
+  // Each test is given 90 seconds.
+  timeout: 90000,
+  // Failing tests will be retried at most two times.
+  retries: 2,
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        browserName: 'chromium',
+
+        // Launch options
+        headless: false,
+        slowMo: 50,
+
+        // Context options
+        viewport: { width: 800, height: 600 },
+        ignoreHTTPSErrors: true,
+
+        // Testing options
+        video: 'retain-on-failure',
+      },
+    },
+  ],
+};
+export default config;
+```
+
+### Locally in the test file
+
+With `test.use()` you can override some options for a file, or a `describe` block.
+
+```js
+// my.spec.ts
+import { test, expect } from "@playwright/test";
+
+// Run tests in this file with portrait-like viewport.
+test.use({ viewport: { width: 600, height: 900 } });
+
+test('my test', async ({ page }) => {
+  // Test code goes here.
+});
+```
+
+### Available options
+
+See the full list of launch options in [`browserType.launch()`](https://playwright.dev/docs/api/class-browsertype#browsertypelaunchoptions) documentation.
+
+See the full list of context options in [`browser.newContext()`](https://playwright.dev/docs/api/class-browser#browsernewcontextoptions) documentation.
+
+Available testing options:
+- `screenshot: 'off' | 'on' | 'only-on-failure'` - Whether to capture a screenshot after each test, off by default.
+  - `off` - Do not capture screenshots.
+  - `on` - Capture screenshot after each test.
+  - `only-on-failure` - Capture screenshot after each test failure.
+- `video: 'off' | 'on' | 'retain-on-failure' | 'retry-with-video'` - Whether to record video for each test, off by default.
+  - `off` - Do not record video.
+  - `on` - Record video for each test.
+  - `retain-on-failure`  - Record video for each test, but remove all videos from successful test runs.
+  - `retry-with-video` - Record video only when retrying a test.
+
+Most notable testing options from [Folio documentation][folio]:
+- `reporter: 'dot' | 'line' | 'list'` - Choose a reporter: minimalist `dot`, concise `line` or detailed `list`. See [Folio reporters][folio-reporters] for more details.
+- `retries: number` - Each failing test will be retried up to the certain number of times.
+- `testDir: string` - Directory where test runner should search for test files.
+- `timeout: number` - Timeout in milliseconds for each test.
+- `workers: number` - The maximum number of worker processes to run in parallel.
+
+## Skip tests with annotations
+
+The Playwright test runner can annotate tests to skip under certain parameters. This is enabled by [Folio annotations][folio-annotations].
+
+```js
+test("should be skipped on firefox", async ({ page, browserName }) => {
+  test.skip(browserName === "firefox", "optional description for the skip");
+  // Test function
+});
+```
+
+## Run tests in parallel
+
+Tests are run in parallel by default, using multiple worker processes. You can control the parallelism with the `workers` option in the configuration file or from the command line.
+
+```sh
+# Run just a single test at a time - no parallelization
+npx folio --workers=1
+
+# Run up to 10 tests in parallel
+npx folio --workers=10
+```
+
+```js
+// config.ts
+import { PlaywrightTestConfig } from "@playwright/test";
+
+const config: PlaywrightTestConfig = {
+  // No parallelization on CI, default value locally.
+  worker: process.env.CI ? 1 : undefined,
+  projects: [
+    // Your projects go here
+  ],
+};
+export default config;
+```
+
+By default, test runner chooses the number of workers based on available CPUs.
+
+## Reporters
+
+Playwright test runner comes with a few built-in reporters for different needs and ability to provide custom reporters. The easiest way to try out built-in reporters is to pass `--reporter` [command line option](#command-line). Built-in terminal reporters are minimalist `dot`, concise `line` and detailed `list`.
+
+```sh
+npx folio --reporter=line
+npx folio --reporter=dot
+npx folio --reporter=list
+```
+
+Alternatively, you can specify the reporter in the configuration file.
+```js
+// config.ts
+import { PlaywrightTestConfig } from "@playwright/test";
+
+const config: PlaywrightTestConfig = {
+  // Concise 'dot' on CI, more interactive 'list' when running locally
+  reporter: process.env.CI ? 'dot' : 'line',
+  projects: [
+    // Your projects go here
+  ],
+};
+export default config;
+```
+
+### Export JUnit or JSON report
+
+The Playwright test runner includes reporters that produce a JUnit compatible XML file or a JSON file with test results.
+
+```js
+// config.ts
+import { PlaywrightTestConfig } from "@playwright/test";
+
+const config: PlaywrightTestConfig = {
+  reporter: [
+    // Live output to the terminal
+    'list',
+    // JUnit compatible xml report
+    { name: 'junit', outputFile: 'report.xml' },
+    // JSON file with test results
+    { name: 'json', outputFile: 'report.json' },
+  ]
+  projects: [
+    // Your projects go here
+  ],
+};
+export default config;
+```
+
+[folio]: https://github.com/microsoft/folio
+[folio-annotations]: https://github.com/microsoft/folio#annotations
+[folio-cli]: https://github.com/microsoft/folio#command-line
+[folio-reporters]: https://github.com/microsoft/folio#reporters
diff --git a/docs/src/test-runner-examples-js.md b/docs/src/test-runner-examples-js.md
new file mode 100644
index 0000000000..a59802293e
--- /dev/null
+++ b/docs/src/test-runner-examples-js.md
@@ -0,0 +1,133 @@
+---
+id: test-runner-examples
+title: "Examples"
+---
+
+<!-- TOC -->
+
+## Multiple pages
+
+The default `context` argument is a [BrowserContext][browser-context]. Browser contexts are isolated execution environments that can host multiple pages. See [multi-page scenarios](./multi-pages.md) for more examples.
+
+```js
+import { test } from "@playwright/test";
+
+test("tests on multiple web pages", async ({ context }) => {
+  const pageFoo = await context.newPage();
+  const pageBar = await context.newPage();
+  // Test function
+});
+```
+
+## Mobile emulation
+
+`use` section in the configuration file can be used to configure mobile emulation in the default `context`.
+
+```js
+// config.ts
+import { PlaywrightTestConfig } from "@playwright/test";
+import { devices } from "playwright";
+
+const config: PlaywrightTestConfig = {
+  timeout: 30000,
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        browserName: 'chromium',
+        headless: true,
+        ...devices["Pixel 2"],
+      },
+    },
+  ],
+};
+export default config;
+```
+
+## Network mocking
+
+Define a custom route that mocks network calls for a browser context.
+
+```js
+// In foo.spec.ts
+import { test, expect } from "@playwright/test";
+
+test.beforeEach(async ({ context }) => {
+  // Block any css requests for each test in this file.
+  await context.route(/.css/, route => route.abort());
+});
+
+test("loads page without css", async ({ page }) => {
+  // Alternatively, block any png requests just for this test.
+  await page.route(/.png/, route => route.abort());
+
+  // Test function code.
+  await page.goto("https://stackoverflow.com");
+});
+```
+
+## Visual comparisons
+
+The `expect` API supports visual comparisons with `toMatchSnapshot`. This uses the [pixelmatch](https://github.com/mapbox/pixelmatch) library, and you can pass `threshold` as an option.
+
+```js
+import { test, expect } from "@playwright/test";
+
+test("compares page screenshot", async ({ page }) => {
+  await page.goto("https://stackoverflow.com");
+  const screenshot = await page.screenshot();
+  expect(screenshot).toMatchSnapshot(`test.png`, { threshold: 0.2 });
+});
+```
+
+On first execution, this will generate golden snapshots. Subsequent runs will compare against the golden snapshots. To update golden snapshots with new actual values, run with the `--update-snapshots` flag.
+
+```sh
+# Update golden snapshots when they differ from actual
+npx folio --update-snapshots
+```
+
+### Page object model
+
+To introduce a Page Object for a particular page, create a class that will use the `page` object.
+
+Create a `LoginPage` helper class to encapsulate common operations on the login page.
+```js
+// login-page.ts
+import type { Page } from "playwright";
+
+export class LoginPage {
+  page: Page;
+
+  constructor(page: Page) {
+    this.page = page;
+  }
+
+  async goto() {
+    await this.page.goto("https://example.com/login");
+  }
+
+  async login() {
+    await this.page.fill("#username", TEST_USERNAME);
+    await this.page.fill("#password", TEST_PASSWORD);
+    await this.page.click("text=Login");
+  }
+}
+```
+
+Use the `LoginPage` class in the tests.
+```js
+// my.spec.ts
+import { test, expect } from "@playwright/test";
+import { LoginPage } from "./login-page";
+
+test('login works', async ({ page }) => {
+  // Create the login page and perform operations.
+  const loginPage = new LoginPage(page);
+  await loginPage.goto();
+  await loginPage.login();
+
+  // Verify it worked.
+  expect(await page.textContent("#user-info")).toBe("Welcome, Test User!");
+});
+```
diff --git a/docs/src/test-runner-intro-js.md b/docs/src/test-runner-intro-js.md
new file mode 100644
index 0000000000..c24cd1554a
--- /dev/null
+++ b/docs/src/test-runner-intro-js.md
@@ -0,0 +1,198 @@
+---
+id: test-runner-intro
+title: "Playwright Tests"
+---
+
+Playwright Test Runner was created specifically to accommodate the needs of the end-to-end testing. It does everything you would expect from the regular test runner, and more. Playwright test allows to:
+
+- Run tests across all browsers.
+- Execute tests in parallel.
+- Enjoy context isolation out of the box.
+- Capture videos, screenshots and other artifacts on failure.
+- Integrate your POMs as extensible fixtures.
+
+There are many more exciting features, so read on!
+
+<!-- TOC -->
+
+Note that you can still wire Playwright to your existing JavaScript [test runner]('./test-runners-other-js.md).
+
+## Installation
+
+```sh
+npm i -D @playwright/test@1.0.0-alpha
+```
+
+## First test
+
+Create `tests/foo.spec.ts` to define your test.
+
+```js
+import { test, expect } from '@playwright/test';
+
+test('is a basic test with the page', async ({ page }) => {
+  await page.goto('https://playwright.dev/');
+  const name = await page.innerText('.navbar__title');
+  expect(name).toBe('Playwright');
+});
+```
+
+Now run your tests:
+
+```sh
+# Assuming that test files are in the tests directory.
+npx folio -c tests
+```
+
+## Test fixtures
+
+You noticed an argument `{ page }` that the test above has access to:
+
+```js
+test('basic test', async ({ page }) => {
+  ...
+```
+
+We call these arguments `fixtures`. Playwright Test comes loaded with those fixtures, and you can add your own fixtures as well. Here is a list of the pre-defined fixtures that you are likely to use most of the time:
+
+- `page`: [Page] - Isolated page for this test run.
+- `context`: [BrowserContext] - Isolated context for this test run. The `page` fixture belongs to this context as well. Learn how to [configure context](#modify-options) below.
+- `browser`: [Browser] - Browsers are shared across tests to optimize resources. Learn how to [configure browser](#modify-options) below.
+- `browserName` - The name of the browser currently running the test. Either `chromium`, `firefox` or `webkit`.
+
+## Test and assertion features
+
+### Focus or skip tests
+
+```js
+import { test, expect } from '@playwright/test';
+
+// You can focus single test.
+test.only('focus this test', async ({ page }) => {
+  // Only this test in the entire project runs.
+});
+
+// You can skip tests.
+test.skip('skip this test', async ({ page }) => {
+});
+```
+
+### Group tests together
+
+```js
+import { test, expect } from '@playwright/test';
+
+test.describe('two tests', () => {
+  test.only('one', async ({ page }) => {
+    // ...
+  });
+
+  test.skip('two', async ({ page }) => {
+    // ...
+  });
+});
+```
+
+### Use test hooks
+
+You can use `test.beforeAll` and `test.afterAll` hooks to set up and tear down resources shared between tests.
+And you can use `test.beforeEach` and `test.afterEach` hooks to set up and tear down resources for each test individually.
+
+```js
+import { test, expect } from '@playwright/test';
+
+test.describe('feature foo', () => {
+  test.beforeEach(async ({ page }) => {
+    // Go to the starting url before each test.
+    await page.goto('https://my.start.url');
+  });
+
+  test('my test', async ({ page }) => {
+    // Assertions use the expect API.
+    expect(page.url()).toBe('https://my.start.url');
+  });
+});
+```
+
+## Write a configuration file
+
+Create `config.ts` to configure your tests: specify browser launch options, run tests in multiple browsers and much more. Here is an example configuration that runs every test in Chromium, Firefox and WebKit.
+
+```js
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  timeout: 30000,  // Each test is given 30 seconds.
+
+  // A project per browser, each running all the tests.
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        browserName: 'chromium',
+        headless: true,
+        viewport: { width: 1280, height: 720 },
+      },
+    },
+
+    {
+      name: 'webkit',
+      use: {
+        browserName: 'webkit',
+        headless: true,
+        viewport: { width: 1280, height: 720 },
+      },
+    },
+
+    {
+      name: 'firefox',
+      use: {
+        browserName: 'firefox',
+        headless: true,
+        viewport: { width: 1280, height: 720 },
+      },
+    }
+  ],
+};
+export default config;
+```
+
+## Run the test suite
+
+Tests can be run in single or multiple browsers, in parallel or sequentially.
+
+```sh
+# Run all tests across Chromium, Firefox and WebKit
+$ npx folio --config=config.ts
+
+# Run tests on a single browser
+$ npx folio --config=config.ts --project=chromium
+
+# Run tests sequentially
+$ npx folio --config=config.ts --workers=1
+
+# Retry failing tests
+$ npx folio --config=config.ts --retries=2
+
+# See all options
+$ npx folio --help
+```
+
+Refer to the [command line documentation][folio-cli] for all options.
+
+### Configure NPM scripts
+
+Save the run command as an NPM script.
+
+```json
+{
+  "scripts": {
+    "test": "npx folio --config=config.ts"
+  }
+}
+```
+
+[folio]: https://github.com/microsoft/folio
+[folio-annotations]: https://github.com/microsoft/folio#annotations
+[folio-cli]: https://github.com/microsoft/folio#command-line
+[folio-reporters]: https://github.com/microsoft/folio#reporters
diff --git a/docs/src/test-runners-js.md b/docs/src/test-runners-js.md
index 54a60a9f66..be5140d575 100644
--- a/docs/src/test-runners-js.md
+++ b/docs/src/test-runners-js.md
@@ -1,12 +1,16 @@
 ---
 id: test-runners
-title: "Test Runners"
+title: "Third party runners"
 ---
 
-With a few lines of code, you can hook up Playwright to your favorite JavaScript test runner.
+With a few lines of code, you can hook up Playwright to your existing JavaScript test runner.
 
 <!-- TOC -->
 
+## @playwright/test
+
+[@playwright/test](./test-runner-intro.md) is our first-party recommended test runner to be used with Playwright. Learn more about it [here](./test-runner-intro.md).
+
 ## Jest / Jasmine
 
 For Jest, [jest-playwright](https://github.com/playwright-community/jest-playwright) can be used. However for a light-weight solution, requiring playwright directly works fine. Jest shares it's syntax with Jasmine, so this applies to Jasmine as well.