From dd0b089d13caed0a180c92b07e4a643ecedee0f4 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Thu, 29 Jul 2021 14:03:58 -0700 Subject: [PATCH] feat(test runner): createContext fixture for multi-context scenarios (#7779) --- docs/src/test-api/class-fixtures.md | 60 ++++++++++++++++++ src/server/trace/recorder/tracing.ts | 2 + src/test/index.ts | 81 ++++++++++++++---------- tests/playwright-test/playwright.spec.ts | 41 +++++++++++- tests/playwright-test/shard.spec.ts | 12 ++-- types/test.d.ts | 65 +++++++++++++++++++ utils/generate_types/overrides-test.d.ts | 1 + 7 files changed, 222 insertions(+), 40 deletions(-) diff --git a/docs/src/test-api/class-fixtures.md b/docs/src/test-api/class-fixtures.md index 1b9cda5791..7b5c45a4e2 100644 --- a/docs/src/test-api/class-fixtures.md +++ b/docs/src/test-api/class-fixtures.md @@ -130,6 +130,66 @@ Learn how to [configure context](./test-configuration.md) through other fixtures The [`property: Fixtures.page`] belongs to this context. +## property: Fixtures.createContext +- type: <[function]\([BrowserContextOptions]|[void]\):[BrowserContext]> + +A function that creates a new context, taking into account all options set +through [configuration file](./test-configuration.md) or [`method: Test.use`] calls. All contexts created by this function are similar to the default [`property: Fixtures.context`]. + +This function is useful for multi-context scenarios, for example testing +two users talking over the chat application. + +A single `options` argument will be merged with all the default options from [configuration file](./test-configuration.md) or [`method: Test.use`] calls and passed to [`method: Browser.newContext`]. If you'd like to undo some of these options, override them with some value or `undefined`. For example: + +```js js-flavor=ts +// example.spec.ts + +import { test } from '@playwright/test'; + +// All contexts will use this storage state. +test.use({ storageState: 'state.json' }); + +test('my test', async ({ createContext }) => { + // An isolated context + const context1 = await createContext(); + + // Another isolated context with custom options + const context2 = await createContext({ + // Undo 'state.json' from above + storageState: undefined, + // Set custom locale + locale: 'en-US', + }); + + // ... +}); +``` + +```js js-flavor=js +// example.spec.js +// @ts-check + +const { test } = require('@playwright/test'); + +// All contexts will use this storage state. +test.use({ storageState: 'state.json' }); + +test('my test', async ({ createContext }) => { + // An isolated context + const context1 = await createContext(); + + // Another isolated context with custom options + const context2 = await createContext({ + // Undo 'state.json' from above + storageState: undefined, + // Set custom locale + locale: 'en-US', + }); + + // ... +}); +``` + ## property: Fixtures.contextOptions - type: <[Object]> diff --git a/src/server/trace/recorder/tracing.ts b/src/server/trace/recorder/tracing.ts index fe28a4fd9c..092ecddeb3 100644 --- a/src/server/trace/recorder/tracing.ts +++ b/src/server/trace/recorder/tracing.ts @@ -61,6 +61,8 @@ export class Tracing implements InstrumentationListener { if (this._recordingTraceEvents) throw new Error('Tracing has already been started'); this._recordingTraceEvents = true; + // TODO: passing the same name for two contexts makes them write into a single file + // and conflict. this._traceFile = path.join(this._tracesDir, (options.name || createGuid()) + '.trace'); this._appendEventChain = mkdirIfNeeded(this._traceFile); diff --git a/src/test/index.ts b/src/test/index.ts index 653c8c4b71..6668a35e4a 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -17,7 +17,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; -import type { LaunchOptions, BrowserContextOptions, Page } from '../../types/types'; +import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext } from '../../types/types'; import type { TestType, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions } from '../../types/test'; import { rootTestType } from './testType'; import { createGuid, removeFolders } from '../utils/utils'; @@ -79,7 +79,7 @@ export const test = _baseTest.extend { + createContext: async ({ browser, screenshot, trace, video, acceptDownloads, bypassCSP, colorScheme, deviceScaleFactor, extraHTTPHeaders, hasTouch, geolocation, httpCredentials, ignoreHTTPSErrors, isMobile, javaScriptEnabled, locale, offline, permissions, proxy, storageState, viewport, timezoneId, userAgent, baseURL, contextOptions }, use, testInfo) => { testInfo.snapshotSuffix = process.platform; if (process.env.PWDEBUG) testInfo.setTimeout(0); @@ -93,17 +93,7 @@ export const test = _baseTest.extend allPages.push(page)); - if (captureTrace) { - const name = path.relative(testInfo.project.outputDir, testInfo.outputDir).replace(/[\/\\]/g, '-'); - await context.tracing.start({ name, screenshots: true, snapshots: true }); - } + await use(async (additionalOptions = {}) => { + let recordVideoDir: string | null = null; + const recordVideoSize = typeof video === 'string' ? undefined : video.size; + if (captureVideo) { + await fs.promises.mkdir(artifactsFolder, { recursive: true }); + recordVideoDir = artifactsFolder; + } - await use(context); + const combinedOptions = { + recordVideo: recordVideoDir ? { dir: recordVideoDir, size: recordVideoSize } : undefined, + ...contextOptions, + ...options, + ...additionalOptions, + }; + const context = await browser.newContext(combinedOptions); + context.setDefaultTimeout(0); + context.on('page', page => allPages.push(page)); + + if (captureTrace) { + const name = path.relative(testInfo.project.outputDir, testInfo.outputDir).replace(/[\/\\]/g, '-'); + const suffix = allContexts.length ? '-' + allContexts.length : ''; + await context.tracing.start({ name: name + suffix, screenshots: true, snapshots: true }); + } + + allContexts.push(context); + return context; + }); const testFailed = testInfo.status !== testInfo.expectedStatus; - const preserveTrace = captureTrace && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1)); - if (preserveTrace) { - const tracePath = testInfo.outputPath(`trace.zip`); - await context.tracing.stop({ path: tracePath }); - testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' }); - } else if (captureTrace) { - await context.tracing.stop(); - } + await Promise.all(allContexts.map(async (context, contextIndex) => { + const preserveTrace = captureTrace && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1)); + if (preserveTrace) { + const suffix = contextIndex ? '-' + contextIndex : ''; + const tracePath = testInfo.outputPath(`trace${suffix}.zip`); + await context.tracing.stop({ path: tracePath }); + testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' }); + } else if (captureTrace) { + await context.tracing.stop(); + } + })); const captureScreenshots = (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed)); if (captureScreenshots) { await Promise.all(allPages.map(async (page, index) => { - const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${++index}.png`); + const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${index + 1}.png`); try { await page.screenshot({ timeout: 5000, path: screenshotPath }); testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' }); @@ -180,8 +192,9 @@ export const test = _baseTest.extend context.close())); if (prependToError) { if (!testInfo.error) { testInfo.error = { value: prependToError }; @@ -209,6 +222,10 @@ export const test = _baseTest.extend { + await use(await createContext()); + }, + page: async ({ context }, use) => { await use(await context.newPage()); }, diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.ts index 74a0803f16..1d51af04e8 100644 --- a/tests/playwright-test/playwright.spec.ts +++ b/tests/playwright-test/playwright.spec.ts @@ -221,6 +221,8 @@ test('should work with screenshot: only-on-failure', async ({ runInlineTest }, t }); test('fail', async ({ page }) => { await page.setContent('
FAIL
'); + const page2 = await page.context().newPage(); + await page2.setContent('
FAIL
'); test.expect(1 + 1).toBe(1); }); `, @@ -230,9 +232,11 @@ test('should work with screenshot: only-on-failure', async ({ runInlineTest }, t expect(result.passed).toBe(1); expect(result.failed).toBe(1); const screenshotPass = testInfo.outputPath('test-results', 'a-pass-chromium', 'test-failed-1.png'); - const screenshotFail = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-1.png'); + const screenshotFail1 = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-1.png'); + const screenshotFail2 = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-2.png'); expect(fs.existsSync(screenshotPass)).toBe(false); - expect(fs.existsSync(screenshotFail)).toBe(true); + expect(fs.existsSync(screenshotFail1)).toBe(true); + expect(fs.existsSync(screenshotFail2)).toBe(true); }); test('should work with video: retain-on-failure', async ({ runInlineTest }, testInfo) => { @@ -327,3 +331,36 @@ test('should work with video size', async ({ runInlineTest }, testInfo) => { expect(videoPlayer.videoWidth).toBe(220); expect(videoPlayer.videoHeight).toBe(110); }); + +test('should work with multiple contexts and trace: on', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { use: { trace: 'on' } }; + `, + 'a.test.ts': ` + const { test } = pwt; + test('pass', async ({ page, createContext }) => { + await page.setContent('
PASS
'); + + const context1 = await createContext(); + const page1 = await context1.newPage(); + await page1.setContent('
PASS
'); + + const context2 = await createContext({ locale: 'en-US' }); + const page2 = await context2.newPage(); + await page2.setContent('
PASS
'); + + test.expect(1 + 1).toBe(2); + }); + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + const traceDefault = testInfo.outputPath('test-results', 'a-pass', 'trace.zip'); + const trace1 = testInfo.outputPath('test-results', 'a-pass', 'trace-1.zip'); + const trace2 = testInfo.outputPath('test-results', 'a-pass', 'trace-2.zip'); + expect(fs.existsSync(traceDefault)).toBe(true); + expect(fs.existsSync(trace1)).toBe(true); + expect(fs.existsSync(trace2)).toBe(true); +}); diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.ts index a75d26353e..1fc2bcaaae 100644 --- a/tests/playwright-test/shard.spec.ts +++ b/tests/playwright-test/shard.spec.ts @@ -50,8 +50,8 @@ test('should respect shard=1/2', async ({ runInlineTest }) => { expect(result.passed).toBe(3); expect(result.skipped).toBe(0); expect(result.output).toContain('test2-done'); - expect(result.output).toContain('test4-done'); - expect(result.output).toContain('test5-done'); + expect(result.output).toContain('test1-done'); + expect(result.output).toContain('test3-done'); }); test('should respect shard=2/2', async ({ runInlineTest }) => { @@ -59,8 +59,8 @@ test('should respect shard=2/2', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); expect(result.passed).toBe(2); expect(result.skipped).toBe(0); - expect(result.output).toContain('test1-done'); - expect(result.output).toContain('test3-done'); + expect(result.output).toContain('test4-done'); + expect(result.output).toContain('test5-done'); }); test('should respect shard=1/2 in config', async ({ runInlineTest }) => { @@ -74,6 +74,6 @@ test('should respect shard=1/2 in config', async ({ runInlineTest }) => { expect(result.passed).toBe(3); expect(result.skipped).toBe(0); expect(result.output).toContain('test2-done'); - expect(result.output).toContain('test4-done'); - expect(result.output).toContain('test5-done'); + expect(result.output).toContain('test1-done'); + expect(result.output).toContain('test3-done'); }); diff --git a/types/test.d.ts b/types/test.d.ts index 3a7352633b..9a92bb90ec 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -2675,6 +2675,71 @@ export interface PlaywrightWorkerArgs { * */ export interface PlaywrightTestArgs { + /** + * A function that creates a new context, taking into account all options set through + * [configuration file](https://playwright.dev/docs/test-configuration) or + * [test.use(fixtures)](https://playwright.dev/docs/api/class-test#test-use) calls. All contexts created by this function + * are similar to the default [fixtures.context](https://playwright.dev/docs/api/class-fixtures#fixtures-context). + * + * This function is useful for multi-context scenarios, for example testing two users talking over the chat application. + * + * A single `options` argument will be merged with all the default options from + * [configuration file](https://playwright.dev/docs/test-configuration) or + * [test.use(fixtures)](https://playwright.dev/docs/api/class-test#test-use) calls and passed to + * [browser.newContext([options])](https://playwright.dev/docs/api/class-browser#browser-new-context). If you'd like to + * undo some of these options, override them with some value or `undefined`. For example: + * + * ```js js-flavor=ts + * // example.spec.ts + * + * import { test } from '@playwright/test'; + * + * // All contexts will use this storage state. + * test.use({ storageState: 'state.json' }); + * + * test('my test', async ({ createContext }) => { + * // An isolated context + * const context1 = await createContext(); + * + * // Another isolated context with custom options + * const context2 = await createContext({ + * // Undo 'state.json' from above + * storageState: undefined, + * // Set custom locale + * locale: 'en-US', + * }); + * + * // ... + * }); + * ``` + * + * ```js js-flavor=js + * // example.spec.js + * // @ts-check + * + * const { test } = require('@playwright/test'); + * + * // All contexts will use this storage state. + * test.use({ storageState: 'state.json' }); + * + * test('my test', async ({ createContext }) => { + * // An isolated context + * const context1 = await createContext(); + * + * // Another isolated context with custom options + * const context2 = await createContext({ + * // Undo 'state.json' from above + * storageState: undefined, + * // Set custom locale + * locale: 'en-US', + * }); + * + * // ... + * }); + * ``` + * + */ + createContext: (options?: BrowserContextOptions) => Promise; /** * Isolated [BrowserContext] instance, created for each test. Since contexts are isolated between each other, every test * gets a fresh environment, even when multiple tests run in a single [Browser] for maximum efficiency. diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 763c60a62f..94d4d0c345 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -317,6 +317,7 @@ export interface PlaywrightWorkerArgs { } export interface PlaywrightTestArgs { + createContext: (options?: BrowserContextOptions) => Promise; context: BrowserContext; page: Page; }