diff --git a/src/client/browser.ts b/src/client/browser.ts index 8860420c2b..47a087c1a6 100644 --- a/src/client/browser.ts +++ b/src/client/browser.ts @@ -23,12 +23,14 @@ import { BrowserContextOptions } from './types'; import { isSafeCloseError } from '../utils/errors'; import * as api from '../../types/types'; import { CDPSession } from './cdpSession'; +import type { BrowserType } from './browserType'; export class Browser extends ChannelOwner implements api.Browser { readonly _contexts = new Set(); private _isConnected = true; private _closedPromise: Promise; _remoteType: 'owns-connection' | 'uses-connection' | null = null; + private _browserType!: BrowserType; readonly _name: string; static from(browser: channels.BrowserChannel): Browser { @@ -46,13 +48,22 @@ export class Browser extends ChannelOwner this.once(Events.Browser.Disconnected, f)); } + _setBrowserType(browserType: BrowserType) { + this._browserType = browserType; + for (const context of this._contexts) + context._setBrowserType(browserType); + } + async newContext(options: BrowserContextOptions = {}): Promise { return this._wrapApiCall(async (channel: channels.BrowserChannel) => { + options = { ...this._browserType._defaultContextOptions, ...options }; const contextOptions = await prepareBrowserContextParams(options); const context = BrowserContext.from((await channel.newContext(contextOptions)).context); context._options = contextOptions; this._contexts.add(context); context._logger = options.logger || this._logger; + context._setBrowserType(this._browserType); + await this._browserType._onDidCreateContext?.(context); return context; }); } diff --git a/src/client/browserContext.ts b/src/client/browserContext.ts index 1323a4c943..5f477018b4 100644 --- a/src/client/browserContext.ts +++ b/src/client/browserContext.ts @@ -33,11 +33,13 @@ import * as api from '../../types/types'; import * as structs from '../../types/structs'; import { CDPSession } from './cdpSession'; import { Tracing } from './tracing'; +import type { BrowserType } from './browserType'; export class BrowserContext extends ChannelOwner implements api.BrowserContext { _pages = new Set(); private _routes: { url: URLMatch, handler: network.RouteHandler }[] = []; readonly _browser: Browser | null = null; + private _browserType: BrowserType | undefined; readonly _bindings = new Map any>(); _timeoutSettings = new TimeoutSettings(); _ownerPage: Page | undefined; @@ -89,6 +91,11 @@ export class BrowserContext extends ChannelOwner this.once(Events.BrowserContext.Close, f)); } + _setBrowserType(browserType: BrowserType) { + this._browserType = browserType; + browserType._contexts.add(this); + } + private _onPage(page: Page): void { this._pages.add(page); this.emit(Events.BrowserContext.Page, page); @@ -311,12 +318,14 @@ export class BrowserContext extends ChannelOwner { try { await this._wrapApiCall(async (channel: channels.BrowserContextChannel) => { + await this._browserType?._onWillCloseContext?.(this); await channel.close(); await this._closedPromise; }); diff --git a/src/client/browserType.ts b/src/client/browserType.ts index beab7ac261..4fdf13c61c 100644 --- a/src/client/browserType.ts +++ b/src/client/browserType.ts @@ -18,7 +18,7 @@ import * as channels from '../protocol/channels'; import { Browser } from './browser'; import { BrowserContext, prepareBrowserContextParams } from './browserContext'; import { ChannelOwner } from './channelOwner'; -import { LaunchOptions, LaunchServerOptions, ConnectOptions, LaunchPersistentContextOptions } from './types'; +import { LaunchOptions, LaunchServerOptions, ConnectOptions, LaunchPersistentContextOptions, BrowserContextOptions } from './types'; import WebSocket from 'ws'; import { Connection } from './connection'; import { Events } from './events'; @@ -45,6 +45,13 @@ export interface BrowserServer extends api.BrowserServer { export class BrowserType extends ChannelOwner implements api.BrowserType { private _timeoutSettings = new TimeoutSettings(); _serverLauncher?: BrowserServerLauncher; + _contexts = new Set(); + + // Instrumentation. + _defaultContextOptions: BrowserContextOptions = {}; + _defaultLaunchOptions: LaunchOptions = {}; + _onDidCreateContext?: (context: BrowserContext) => Promise; + _onWillCloseContext?: (context: BrowserContext) => Promise; static from(browserType: channels.BrowserTypeChannel): BrowserType { return (browserType as any)._object; @@ -69,6 +76,7 @@ export class BrowserType extends ChannelOwner { assert(!(options as any).userDataDir, 'userDataDir option is not supported in `browserType.launch`. Use `browserType.launchPersistentContext` instead'); assert(!(options as any).port, 'Cannot specify a port without launching as a server.'); + options = { ...this._defaultLaunchOptions, ...options }; const launchOptions: channels.BrowserTypeLaunchParams = { ...options, ignoreDefaultArgs: Array.isArray(options.ignoreDefaultArgs) ? options.ignoreDefaultArgs : undefined, @@ -77,6 +85,7 @@ export class BrowserType extends ChannelOwner { return this._wrapApiCall(async (channel: channels.BrowserTypeChannel) => { assert(!(options as any).port, 'Cannot specify a port without launching as a server.'); + options = { ...this._defaultLaunchOptions, ...this._defaultContextOptions, ...options }; const contextParams = await prepareBrowserContextParams(options); const persistentParams: channels.BrowserTypeLaunchPersistentContextParams = { ...contextParams, @@ -103,6 +113,8 @@ export class BrowserType extends ChannelOwner { // Emulate all pages, contexts and the browser closing upon disconnect. for (const context of browser.contexts()) { @@ -252,6 +265,7 @@ export class BrowserType extends ChannelOwner = this; + while (!ancestorWithCSI._csi && ancestorWithCSI._parent) + ancestorWithCSI = ancestorWithCSI._parent; let csiCallback: ((e?: Error) => void) | undefined; + try { logApiCall(logger, `=> ${apiName} started`); - csiCallback = this._csi?.onApiCall(apiName); + csiCallback = ancestorWithCSI._csi?.onApiCall(apiName); const result = await func(channel as any, stackTrace); csiCallback?.(); logApiCall(logger, `<= ${apiName} succeeded`); diff --git a/src/client/page.ts b/src/client/page.ts index cf7fa8eb02..b9ae4833fe 100644 --- a/src/client/page.ts +++ b/src/client/page.ts @@ -487,9 +487,10 @@ export class Page extends ChannelOwner { - await channel.close(options); if (this._ownedContext) await this._ownedContext.close(); + else + await channel.close(options); }); } catch (e) { if (isSafeCloseError(e)) diff --git a/src/test/index.ts b/src/test/index.ts index b7230ee297..0e26538823 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -16,27 +16,51 @@ import * as fs from 'fs'; import * as path from 'path'; -import * as os from 'os'; -import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext } from '../../types/types'; +import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext, BrowserType } from '../../types/types'; import type { TestType, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions } from '../../types/test'; import { rootTestType } from './testType'; import { createGuid, removeFolders } from '../utils/utils'; export { expect } from './expect'; export const _baseTest: TestType<{}, {}> = rootTestType.test; -const artifactsFolder = path.join(os.tmpdir(), 'pwt-' + createGuid()); +type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & { + _combinedContextOptions: BrowserContextOptions, + _setupContextOptionsAndArtifacts: void; +}; +type WorkerAndFileFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & { + _browserType: BrowserType; + _artifactsDir: () => string, +}; -export const test = _baseTest.extend({ +export const test = _baseTest.extend({ defaultBrowserType: [ 'chromium', { scope: 'worker' } ], browserName: [ ({ defaultBrowserType }, use) => use(defaultBrowserType), { scope: 'worker' } ], playwright: [ require('../inprocess'), { scope: 'worker' } ], headless: [ undefined, { scope: 'worker' } ], channel: [ undefined, { scope: 'worker' } ], launchOptions: [ {}, { scope: 'worker' } ], + screenshot: [ 'off', { scope: 'worker' } ], + video: [ 'off', { scope: 'worker' } ], + trace: [ 'off', { scope: 'worker' } ], - browser: [ async ({ playwright, browserName, headless, channel, launchOptions }, use) => { + _artifactsDir: [async ({}, use, workerInfo) => { + let dir: string | undefined; + await use(() => { + if (!dir) { + dir = path.join(workerInfo.project.outputDir, '.playwright-artifacts-' + workerInfo.workerIndex); + fs.mkdirSync(dir, { recursive: true }); + } + return dir; + }); + if (dir) + await removeFolders([dir]); + }, { scope: 'worker' }], + + _browserType: [async ({ playwright, browserName, headless, channel, launchOptions }, use) => { if (!['chromium', 'firefox', 'webkit'].includes(browserName)) throw new Error(`Unexpected browserName "${browserName}", must be one of "chromium", "firefox" or "webkit"`); + const browserType = playwright[browserName]; + const options: LaunchOptions = { handleSIGINT: false, timeout: 0, @@ -46,15 +70,18 @@ export const test = _baseTest.extend { + const browser = await _browserType.launch(); await use(browser); await browser.close(); - await removeFolders([artifactsFolder]); }, { scope: 'worker' } ], - screenshot: 'off', - video: 'off', - trace: 'off', acceptDownloads: undefined, bypassCSP: undefined, colorScheme: undefined, @@ -81,11 +108,7 @@ export const test = _baseTest.extend { - testInfo.snapshotSuffix = process.platform; - if (process.env.PWDEBUG) - testInfo.setTimeout(0); - - let videoMode = typeof video === 'string' ? video : video.mode; - if (videoMode === 'retry-with-video') - videoMode = 'on-first-retry'; - if (trace === 'retry-with-trace') - trace = 'on-first-retry'; - - const captureVideo = (videoMode === 'on' || videoMode === 'retain-on-failure' || (videoMode === 'on-first-retry' && testInfo.retry === 1)); - const captureTrace = (trace === 'on' || trace === 'retain-on-failure' || (trace === 'on-first-retry' && testInfo.retry === 1)); - + }, use) => { const options: BrowserContextOptions = {}; if (acceptDownloads !== undefined) options.acceptDownloads = acceptDownloads; @@ -164,6 +172,124 @@ export const test = _baseTest.extend { + testInfo.snapshotSuffix = process.platform; + if (process.env.PWDEBUG) + testInfo.setTimeout(0); + + if (trace === 'retry-with-trace') + trace = 'on-first-retry'; + const captureTrace = (trace === 'on' || trace === 'retain-on-failure' || (trace === 'on-first-retry' && testInfo.retry === 1)); + const temporaryTraceFiles: string[] = []; + const temporaryScreenshots: string[] = []; + + const onDidCreateContext = async (context: BrowserContext) => { + context.setDefaultTimeout(actionTimeout || 0); + context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0); + if (captureTrace) + await context.tracing.start({ screenshots: true, snapshots: true }); + (context as any)._csi = { + onApiCall: (name: string) => { + return (testInfo as any)._addStep('pw:api', name); + }, + }; + }; + + const onWillCloseContext = async (context: BrowserContext) => { + if (captureTrace) { + // Export trace for now. We'll know whether we have to preserve it + // after the test finishes. + const tracePath = path.join(_artifactsDir(), createGuid() + '.zip'); + temporaryTraceFiles.push(tracePath); + await (context.tracing as any)._export({ path: tracePath }); + } + if (screenshot === 'on' || screenshot === 'only-on-failure') { + // Capture screenshot for now. We'll know whether we have to preserve them + // after the test finishes. + await Promise.all(context.pages().map(async page => { + const screenshotPath = path.join(_artifactsDir(), createGuid() + '.png'); + temporaryScreenshots.push(screenshotPath); + await page.screenshot({ timeout: 5000, path: screenshotPath }).catch(() => {}); + })); + } + }; + + // 1. Setup instrumentation and process existing contexts. + const oldOnDidCreateContext = (_browserType as any)._onDidCreateContext; + (_browserType as any)._onDidCreateContext = onDidCreateContext; + (_browserType as any)._onWillCloseContext = onWillCloseContext; + (_browserType as any)._defaultContextOptions = _combinedContextOptions; + const existingContexts = Array.from((_browserType as any)._contexts) as BrowserContext[]; + await Promise.all(existingContexts.map(onDidCreateContext)); + + // 2. Run the test. + await use(); + + // 3. Determine whether we need the artifacts. + const testFailed = testInfo.status !== testInfo.expectedStatus; + const isHook = testInfo.title === 'beforeAll' || testInfo.title === 'afterAll'; + const preserveTrace = captureTrace && !isHook && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1)); + const captureScreenshots = !isHook && (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed)); + + const traceAttachments: string[] = []; + const addTraceAttachment = () => { + const tracePath = testInfo.outputPath(`trace${traceAttachments.length ? '-' + traceAttachments.length : ''}.zip`); + traceAttachments.push(tracePath); + testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' }); + return tracePath; + }; + + const screenshotAttachments: string[] = []; + const addScreenshotAttachment = () => { + const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${screenshotAttachments.length + 1}.png`); + screenshotAttachments.push(screenshotPath); + testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' }); + return screenshotPath; + }; + + // 4. Cleanup instrumentation. + const leftoverContexts = Array.from((_browserType as any)._contexts) as BrowserContext[]; + (_browserType as any)._onDidCreateContext = oldOnDidCreateContext; + (_browserType as any)._onWillCloseContext = undefined; + (_browserType as any)._defaultContextOptions = undefined; + leftoverContexts.forEach(context => (context as any)._csi = undefined); + + // 5. Collect artifacts from any non-closed contexts. + await Promise.all(leftoverContexts.map(async context => { + if (preserveTrace) + await (context.tracing as any)._export({ path: addTraceAttachment() }); + if (captureScreenshots) + await Promise.all(context.pages().map(page => page.screenshot({ timeout: 5000, path: addScreenshotAttachment() }).catch(() => {}))); + })); + + // 6. Either remove or attach temporary traces and screenshots for contexts closed + // before the test has finished. + await Promise.all(temporaryTraceFiles.map(async file => { + if (preserveTrace) + await fs.promises.rename(file, addTraceAttachment()).catch(() => {}); + else + await fs.promises.unlink(file).catch(() => {}); + })); + await Promise.all(temporaryScreenshots.map(async file => { + if (captureScreenshots) + await fs.promises.rename(file, addScreenshotAttachment()).catch(() => {}); + else + await fs.promises.unlink(file).catch(() => {}); + })); + }, { auto: true }], + + createContext: async ({ browser, video, _artifactsDir }, use, testInfo) => { + let videoMode = typeof video === 'string' ? video : video.mode; + if (videoMode === 'retry-with-video') + videoMode = 'on-first-retry'; + + const captureVideo = (videoMode === 'on' || videoMode === 'retain-on-failure' || (videoMode === 'on-first-retry' && testInfo.retry === 1)); const allContexts: BrowserContext[] = []; const allPages: Page[] = []; @@ -171,64 +297,21 @@ export const test = _baseTest.extend { let recordVideoDir: string | null = null; const recordVideoSize = typeof video === 'string' ? undefined : video.size; - if (captureVideo) { - await fs.promises.mkdir(artifactsFolder, { recursive: true }); - recordVideoDir = artifactsFolder; - } + if (captureVideo) + recordVideoDir = _artifactsDir(); const combinedOptions: BrowserContextOptions = { recordVideo: recordVideoDir ? { dir: recordVideoDir, size: recordVideoSize } : undefined, - ...contextOptions, - ...options, ...additionalOptions, }; const context = await browser.newContext(combinedOptions); - (context as any)._csi = { - onApiCall: (name: string) => { - return (testInfo as any)._addStep('pw:api', name); - }, - }; - context.setDefaultTimeout(actionTimeout || 0); - context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0); context.on('page', page => allPages.push(page)); - if (captureTrace) { - const name = path.relative(testInfo.project.outputDir, testInfo.outputDir).replace(/[\/\\]/g, '-'); - const suffix = allContexts.length ? '-' + allContexts.length : ''; - await context.tracing.start({ name: name + suffix, screenshots: true, snapshots: true }); - } - allContexts.push(context); return context; }); - const testFailed = testInfo.status !== testInfo.expectedStatus; - - await Promise.all(allContexts.map(async (context, contextIndex) => { - const preserveTrace = captureTrace && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1)); - if (preserveTrace) { - const suffix = contextIndex ? '-' + contextIndex : ''; - const tracePath = testInfo.outputPath(`trace${suffix}.zip`); - await context.tracing.stop({ path: tracePath }); - testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' }); - } else if (captureTrace) { - await context.tracing.stop(); - } - })); - - const captureScreenshots = (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed)); - if (captureScreenshots) { - await Promise.all(allPages.map(async (page, index) => { - const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${index + 1}.png`); - try { - await page.screenshot({ timeout: 5000, path: screenshotPath }); - testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' }); - } catch { - } - })); - } - - const prependToError = (testInfo.status === 'timedOut' && allContexts.length) ? + const prependToError = (testInfo.status === 'timedOut' && allContexts.length) ? formatPendingCalls((allContexts[0] as any)._connection.pendingProtocolCalls()) : ''; await Promise.all(allContexts.map(context => context.close())); if (prependToError) { @@ -241,6 +324,7 @@ export const test = _baseTest.extend { @@ -259,7 +343,9 @@ export const test = _baseTest.extend { + context: async ({ createContext }, use, testInfo) => { + if (testInfo.title === 'beforeAll' || testInfo.title === 'afterAll') + throw new Error(`"context" and "page" fixtures are not suppoted in ${testInfo.title}. Use browser.newContext() instead.`); await use(await createContext()); }, diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index 34103842dd..588fb3375f 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -19,11 +19,23 @@ import * as fs from 'fs'; import * as path from 'path'; import { test, expect, stripAscii } from './playwright-test-fixtures'; +const files = { + 'helper.ts': ` + export const test = pwt.test.extend({ + auto: [ async ({}, run, testInfo) => { + testInfo.snapshotSuffix = ''; + await run(); + }, { auto: true } ] + }); + ` +}; + test('should support golden', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.txt': `Hello world`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').toMatchSnapshot('snapshot.txt'); }); @@ -34,6 +46,7 @@ test('should support golden', async ({runInlineTest}) => { test('should fail on wrong golden', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.txt': `Line1 Line2 Line3 @@ -42,7 +55,7 @@ Line5 Line6 Line7`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { const data = []; data.push('Line1'); @@ -67,9 +80,10 @@ Line7`, test('should write detailed failure result to an output folder', async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.txt': `Hello world`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world updated').toMatchSnapshot('snapshot.txt'); }); @@ -89,9 +103,10 @@ test('should write detailed failure result to an output folder', async ({runInli test("doesn\'t create comparison artifacts in an output folder for passed negated snapshot matcher", async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.txt': `Hello world`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world updated').not.toMatchSnapshot('snapshot.txt'); }); @@ -110,9 +125,10 @@ test("doesn\'t create comparison artifacts in an output folder for passed negate test('should pass on different snapshots with negate matcher', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.txt': `Hello world`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world updated').not.toMatchSnapshot('snapshot.txt'); }); @@ -124,9 +140,10 @@ test('should pass on different snapshots with negate matcher', async ({runInline test('should fail on same snapshots with negate matcher', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.txt': `Hello world`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').not.toMatchSnapshot('snapshot.txt'); }); @@ -140,8 +157,9 @@ test('should fail on same snapshots with negate matcher', async ({runInlineTest} test('should write missing expectations locally', async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ + ...files, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').toMatchSnapshot('snapshot.txt'); }); @@ -157,8 +175,9 @@ test('should write missing expectations locally', async ({runInlineTest}, testIn test('shouldn\'t write missing expectations locally for negated matcher', async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ + ...files, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').not.toMatchSnapshot('snapshot.txt'); }); @@ -175,9 +194,10 @@ test('should update snapshot with the update-snapshots flag', async ({runInlineT const EXPECTED_SNAPSHOT = 'Hello world'; const ACTUAL_SNAPSHOT = 'Hello world updated'; const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt'); }); @@ -195,9 +215,10 @@ test('shouldn\'t update snapshot with the update-snapshots flag for negated matc const EXPECTED_SNAPSHOT = 'Hello world'; const ACTUAL_SNAPSHOT = 'Hello world updated'; const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('${ACTUAL_SNAPSHOT}').not.toMatchSnapshot('snapshot.txt'); }); @@ -213,8 +234,9 @@ test('shouldn\'t update snapshot with the update-snapshots flag for negated matc test('should silently write missing expectations locally with the update-snapshots flag', async ({runInlineTest}, testInfo) => { const ACTUAL_SNAPSHOT = 'Hello world new'; const result = await runInlineTest({ + ...files, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt'); }); @@ -230,8 +252,9 @@ test('should silently write missing expectations locally with the update-snapsho test('should silently write missing expectations locally with the update-snapshots flag for negated matcher', async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ + ...files, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').not.toMatchSnapshot('snapshot.txt'); }); @@ -246,11 +269,12 @@ test('should silently write missing expectations locally with the update-snapsho test('should match multiple snapshots', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot1.txt': `Snapshot1`, 'a.spec.js-snapshots/snapshot2.txt': `Snapshot2`, 'a.spec.js-snapshots/snapshot3.txt': `Snapshot3`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Snapshot1').toMatchSnapshot('snapshot1.txt'); expect('Snapshot2').toMatchSnapshot('snapshot2.txt'); @@ -263,6 +287,7 @@ test('should match multiple snapshots', async ({runInlineTest}) => { test('should match snapshots from multiple projects', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'playwright.config.ts': ` import * as path from 'path'; module.exports = { projects: [ @@ -271,14 +296,14 @@ test('should match snapshots from multiple projects', async ({runInlineTest}) => ]}; `, 'p1/a.spec.js': ` - const { test } = pwt; + const { test } = require('../helper'); test('is a test', ({}) => { expect('Snapshot1').toMatchSnapshot('snapshot.txt'); }); `, 'p1/a.spec.js-snapshots/snapshot.txt': `Snapshot1`, 'p2/a.spec.js': ` - const { test } = pwt; + const { test } = require('../helper'); test('is a test', ({}) => { expect('Snapshot2').toMatchSnapshot('snapshot.txt'); }); @@ -290,9 +315,10 @@ test('should match snapshots from multiple projects', async ({runInlineTest}) => test('should use provided name', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/provided.txt': `Hello world`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').toMatchSnapshot('provided.txt'); }); @@ -303,8 +329,9 @@ test('should use provided name', async ({runInlineTest}) => { test('should throw without a name', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').toMatchSnapshot(); }); @@ -316,9 +343,10 @@ test('should throw without a name', async ({runInlineTest}) => { test('should use provided name via options', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/provided.txt': `Hello world`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').toMatchSnapshot({ name: 'provided.txt' }); }); @@ -329,9 +357,10 @@ test('should use provided name via options', async ({runInlineTest}) => { test('should compare binary', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.dat': Buffer.from([1,2,3,4]), 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect(Buffer.from([1,2,3,4])).toMatchSnapshot('snapshot.dat'); }); @@ -342,10 +371,11 @@ test('should compare binary', async ({runInlineTest}) => { test('should compare PNG images', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.png': Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'), 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64')).toMatchSnapshot('snapshot.png'); }); @@ -356,10 +386,11 @@ test('should compare PNG images', async ({runInlineTest}) => { test('should compare different PNG images', async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.png': Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'), 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII==', 'base64')).toMatchSnapshot('snapshot.png'); }); @@ -384,10 +415,11 @@ test('should respect threshold', async ({runInlineTest}) => { const expected = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-expected.png')); const actual = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-actual.png')); const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/snapshot.png': expected, 'a.spec.js-snapshots/snapshot2.png': expected, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { threshold: 0.3 }); expect(Buffer.from('${actual.toString('base64')}', 'base64')).not.toMatchSnapshot('snapshot.png', { threshold: 0.2 }); @@ -403,6 +435,7 @@ test('should respect project threshold', async ({runInlineTest}) => { const expected = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-expected.png')); const actual = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-actual.png')); const result = await runInlineTest({ + ...files, 'playwright.config.ts': ` module.exports = { projects: [ { expect: { toMatchSnapshot: { threshold: 0.2 } } }, @@ -411,7 +444,7 @@ test('should respect project threshold', async ({runInlineTest}) => { 'a.spec.js-snapshots/snapshot.png': expected, 'a.spec.js-snapshots/snapshot2.png': expected, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { threshold: 0.3 }); expect(Buffer.from('${actual.toString('base64')}', 'base64')).not.toMatchSnapshot('snapshot.png'); @@ -425,9 +458,10 @@ test('should respect project threshold', async ({runInlineTest}) => { test('should sanitize snapshot name', async ({runInlineTest}) => { const result = await runInlineTest({ + ...files, 'a.spec.js-snapshots/-snapshot-.txt': `Hello world`, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').toMatchSnapshot('../../snapshot!.txt'); }); @@ -438,8 +472,9 @@ test('should sanitize snapshot name', async ({runInlineTest}) => { test('should write missing expectations with sanitized snapshot name', async ({runInlineTest}, testInfo) => { const result = await runInlineTest({ + ...files, 'a.spec.js': ` - const { test } = pwt; + const { test } = require('./helper'); test('is a test', ({}) => { expect('Hello world').toMatchSnapshot('../../snapshot!.txt'); }); diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.ts new file mode 100644 index 0000000000..b00c7c3e18 --- /dev/null +++ b/tests/playwright-test/playwright.artifacts.spec.ts @@ -0,0 +1,280 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; +import fs from 'fs'; +import path from 'path'; + +function listFiles(dir: string): string[] { + const result: string[] = []; + const entries = fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + result.push(entry.name); + if (entry.isDirectory()) + result.push(...listFiles(path.join(dir, entry.name)).map(x => ' ' + x)); + } + return result; +} + +const testFiles = { + 'artifacts.spec.ts': ` + import fs from 'fs'; + import os from 'os'; + import path from 'path'; + import rimraf from 'rimraf'; + + const { test } = pwt; + + test.describe('shared', () => { + let page; + test.beforeAll(async ({ browser }) => { + page = await browser.newPage({}); + await page.setContent(''); + }); + + test.afterAll(async () => { + await page.close(); + }); + + test('shared passing', async ({ }) => { + await page.click('text=Click me'); + }); + + test('shared failing', async ({ }) => { + await page.click('text=And me'); + expect(1).toBe(2); + }); + }); + + test('passing', async ({ page }) => { + await page.setContent('I am the page'); + }); + + test('two contexts', async ({ page, createContext }) => { + await page.setContent('I am the page'); + + const context2 = await createContext(); + const page2 = await context2.newPage(); + await page2.setContent('I am the page'); + }); + + test('failing', async ({ page }) => { + await page.setContent('I am the page'); + expect(1).toBe(2); + }); + + test('two contexts failing', async ({ page, createContext }) => { + await page.setContent('I am the page'); + + const context2 = await createContext(); + const page2 = await context2.newPage(); + await page2.setContent('I am the page'); + + expect(1).toBe(2); + }); + + test('own context passing', async ({ browser }) => { + const page = await browser.newPage(); + await page.setContent(''); + await page.click('text=Click me'); + await page.close(); + }); + + test('own context failing', async ({ browser }) => { + const page = await browser.newPage(); + await page.setContent(''); + await page.click('text=Click me'); + await page.close(); + expect(1).toBe(2); + }); + + const testPersistent = test.extend({ + page: async ({ playwright, browserName }, use) => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'user-data-dir-')); + const context = await playwright[browserName].launchPersistentContext(dir); + await use(context.pages()[0]); + await context.close(); + rimraf.sync(dir); + }, + }); + + testPersistent('persistent passing', async ({ page }) => { + await page.setContent(''); + }); + + testPersistent('persistent failing', async ({ page }) => { + await page.setContent(''); + expect(1).toBe(2); + }); + `, +}; + +test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { use: { screenshot: 'on' } }; + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(5); + expect(result.failed).toBe(5); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + 'artifacts-failing', + ' test-failed-1.png', + 'artifacts-own-context-failing', + ' test-failed-1.png', + 'artifacts-own-context-passing', + ' test-finished-1.png', + 'artifacts-passing', + ' test-finished-1.png', + 'artifacts-persistent-failing', + ' test-failed-1.png', + 'artifacts-persistent-passing', + ' test-finished-1.png', + 'artifacts-shared-failing', + ' test-failed-1.png', + 'artifacts-shared-passing', + ' test-finished-1.png', + 'artifacts-two-contexts', + ' test-finished-1.png', + ' test-finished-2.png', + 'artifacts-two-contexts-failing', + ' test-failed-1.png', + ' test-failed-2.png', + 'report.json', + ]); +}); + +test('should work with screenshot: only-on-failure', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { use: { screenshot: 'only-on-failure' } }; + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(5); + expect(result.failed).toBe(5); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + 'artifacts-failing', + ' test-failed-1.png', + 'artifacts-own-context-failing', + ' test-failed-1.png', + 'artifacts-persistent-failing', + ' test-failed-1.png', + 'artifacts-shared-failing', + ' test-failed-1.png', + 'artifacts-two-contexts-failing', + ' test-failed-1.png', + ' test-failed-2.png', + 'report.json', + ]); +}); + +test('should work with trace: on', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { use: { trace: 'on' } }; + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(5); + expect(result.failed).toBe(5); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + 'artifacts-failing', + ' trace.zip', + 'artifacts-own-context-failing', + ' trace.zip', + 'artifacts-own-context-passing', + ' trace.zip', + 'artifacts-passing', + ' trace.zip', + 'artifacts-persistent-failing', + ' trace.zip', + 'artifacts-persistent-passing', + ' trace.zip', + 'artifacts-shared-failing', + ' trace.zip', + 'artifacts-shared-passing', + ' trace.zip', + 'artifacts-two-contexts', + ' trace-1.zip', + ' trace.zip', + 'artifacts-two-contexts-failing', + ' trace-1.zip', + ' trace.zip', + 'report.json', + ]); +}); + +test('should work with trace: retain-on-failure', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { use: { trace: 'retain-on-failure' } }; + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(5); + expect(result.failed).toBe(5); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + 'artifacts-failing', + ' trace.zip', + 'artifacts-own-context-failing', + ' trace.zip', + 'artifacts-persistent-failing', + ' trace.zip', + 'artifacts-shared-failing', + ' trace.zip', + 'artifacts-two-contexts-failing', + ' trace-1.zip', + ' trace.zip', + 'report.json', + ]); +}); + +test('should work with trace: on-first-retry', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + module.exports = { use: { trace: 'on-first-retry' } }; + `, + }, { workers: 1, retries: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(5); + expect(result.failed).toBe(5); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + 'artifacts-failing-retry1', + ' trace.zip', + 'artifacts-own-context-failing-retry1', + ' trace.zip', + 'artifacts-persistent-failing-retry1', + ' trace.zip', + 'artifacts-shared-failing-retry1', + ' trace.zip', + 'artifacts-two-contexts-failing-retry1', + ' trace-1.zip', + ' trace.zip', + 'report.json', + ]); +}); diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.ts index c4f009325f..2a05920bb6 100644 --- a/tests/playwright-test/playwright.spec.ts +++ b/tests/playwright-test/playwright.spec.ts @@ -161,6 +161,65 @@ test('should override use:browserName with --browser', async ({ runInlineTest }) ]); }); +test('should respect context options in various contexts', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { use: { viewport: { width: 500, height: 500 } } }; + `, + 'a.test.ts': ` + import fs from 'fs'; + import os from 'os'; + import path from 'path'; + import rimraf from 'rimraf'; + + const { test } = pwt; + test.use({ locale: 'fr-CH' }); + + let context; + test.beforeAll(async ({ browser }) => { + context = await browser.newContext(); + }); + + test.afterAll(async () => { + await context.close(); + }); + + test('shared context', async ({}) => { + const page = await context.newPage(); + expect(page.viewportSize()).toEqual({ width: 500, height: 500 }); + expect(await page.evaluate(() => navigator.language)).toBe('fr-CH'); + }); + + test('own context', async ({ browser }) => { + const page = await browser.newPage(); + expect(page.viewportSize()).toEqual({ width: 500, height: 500 }); + expect(await page.evaluate(() => navigator.language)).toBe('fr-CH'); + await page.close(); + }); + + test('default context', async ({ page }) => { + expect(page.viewportSize()).toEqual({ width: 500, height: 500 }); + expect(await page.evaluate(() => navigator.language)).toBe('fr-CH'); + }); + + test('persistent context', async ({ playwright, browserName }) => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'user-data-dir-')); + const context = await playwright[browserName].launchPersistentContext(dir); + const page = context.pages()[0]; + + expect(page.viewportSize()).toEqual({ width: 500, height: 500 }); + expect(await page.evaluate(() => navigator.language)).toBe('fr-CH'); + + await context.close(); + rimraf.sync(dir); + }); + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(4); +}); + test('should report error and pending operations on timeout', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'a.test.ts': ` @@ -185,6 +244,22 @@ test('should report error and pending operations on timeout', async ({ runInline expect(stripAscii(result.output)).toContain(`10 | page.textContent('text=More missing'),`); }); +test('should throw when using page in beforeAll', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + const { test } = pwt; + test.beforeAll(async ({ page }) => { + }); + test('ok', async ({ page }) => { + }); + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(0); + expect(result.output).toContain(`Error: "context" and "page" fixtures are not suppoted in beforeAll. Use browser.newContext() instead.`); +}); + test('should report click error on sigint', async ({ runInlineTest }) => { test.skip(process.platform === 'win32', 'No sending SIGINT on Windows'); @@ -208,37 +283,6 @@ test('should report click error on sigint', async ({ runInlineTest }) => { expect(stripAscii(result.output)).toContain(`8 | const promise = page.click('text=Missing');`); }); -test('should work with screenshot: only-on-failure', async ({ runInlineTest }, testInfo) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { use: { screenshot: 'only-on-failure' }, name: 'chromium' }; - `, - 'a.test.ts': ` - const { test } = pwt; - test('pass', async ({ page }) => { - await page.setContent('
PASS
'); - test.expect(1 + 1).toBe(2); - }); - test('fail', async ({ page }) => { - await page.setContent('
FAIL
'); - const page2 = await page.context().newPage(); - await page2.setContent('
FAIL
'); - test.expect(1 + 1).toBe(1); - }); - `, - }, { workers: 1 }); - - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(1); - expect(result.failed).toBe(1); - const screenshotPass = testInfo.outputPath('test-results', 'a-pass-chromium', 'test-failed-1.png'); - const screenshotFail1 = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-1.png'); - const screenshotFail2 = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-2.png'); - expect(fs.existsSync(screenshotPass)).toBe(false); - expect(fs.existsSync(screenshotFail1)).toBe(true); - expect(fs.existsSync(screenshotFail2)).toBe(true); -}); - test('should work with video: retain-on-failure', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'playwright.config.ts': ` @@ -339,36 +383,3 @@ test('should work with video size', async ({ runInlineTest }, testInfo) => { expect(videoPlayer.videoWidth).toBe(220); expect(videoPlayer.videoHeight).toBe(110); }); - -test('should work with multiple contexts and trace: on', async ({ runInlineTest }, testInfo) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { use: { trace: 'on' } }; - `, - 'a.test.ts': ` - const { test } = pwt; - test('pass', async ({ page, createContext }) => { - await page.setContent('
PASS
'); - - const context1 = await createContext(); - const page1 = await context1.newPage(); - await page1.setContent('
PASS
'); - - const context2 = await createContext({ locale: 'en-US' }); - const page2 = await context2.newPage(); - await page2.setContent('
PASS
'); - - test.expect(1 + 1).toBe(2); - }); - `, - }, { workers: 1 }); - - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - const traceDefault = testInfo.outputPath('test-results', 'a-pass', 'trace.zip'); - const trace1 = testInfo.outputPath('test-results', 'a-pass', 'trace-1.zip'); - const trace2 = testInfo.outputPath('test-results', 'a-pass', 'trace-2.zip'); - expect(fs.existsSync(traceDefault)).toBe(true); - expect(fs.existsSync(trace1)).toBe(true); - expect(fs.existsSync(trace2)).toBe(true); -}); diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index de39471025..677c6c0a18 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -114,7 +114,6 @@ test('should work with custom reporter', async ({ runInlineTest }) => { ]); }); - test('should work without a file extension', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': smallReporterJS, @@ -162,6 +161,9 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => { test('should report expect steps', async ({ runInlineTest }) => { const expectReporterJS = ` class Reporter { + onStdOut(chunk) { + process.stdout.write(chunk); + } onStepBegin(test, result, step) { const copy = { ...step, startTime: undefined, duration: undefined }; console.log('%%%% begin', JSON.stringify(copy)); @@ -232,6 +234,15 @@ test('should report expect steps', async ({ runInlineTest }) => { test('should report api steps', async ({ runInlineTest }) => { const expectReporterJS = ` class Reporter { + onStdOut(chunk) { + process.stdout.write(chunk); + } + onTestBegin(test) { + console.log('%%%% test begin ' + test.title); + } + onTestEnd(test) { + console.log('%%%% test end ' + test.title); + } onStepBegin(test, result, step) { const copy = { ...step, startTime: undefined, duration: undefined }; console.log('%%%% begin', JSON.stringify(copy)); @@ -259,11 +270,31 @@ test('should report api steps', async ({ runInlineTest }) => { await page.setContent(''); await page.click('button'); }); + + test.describe('suite', () => { + let myPage; + test.beforeAll(async ({ browser }) => { + myPage = await browser.newPage(); + await myPage.setContent(''); + }); + + test('pass1', async () => { + await myPage.click('button'); + }); + test('pass2', async () => { + await myPage.click('button'); + }); + + test.afterAll(async () => { + await myPage.close(); + }); + }); ` }, { reporter: '', workers: 1 }); expect(result.exitCode).toBe(0); expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([ + `%%%% test begin pass`, `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, @@ -276,6 +307,23 @@ test('should report api steps', async ({ runInlineTest }) => { `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`, `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%%%% test end pass`, + `%%%% test begin pass1`, + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"page.click\",\"category\":\"pw:api\"}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%%%% test end pass1`, + `%%%% test begin pass2`, + `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`, + `%% end {\"title\":\"page.click\",\"category\":\"pw:api\"}`, + `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `%%%% test end pass2`, ]); }); @@ -283,6 +331,9 @@ test('should report api steps', async ({ runInlineTest }) => { test('should report api step failure', async ({ runInlineTest }) => { const expectReporterJS = ` class Reporter { + onStdOut(chunk) { + process.stdout.write(chunk); + } onStepBegin(test, result, step) { const copy = { ...step, startTime: undefined, duration: undefined }; console.log('%%%% begin', JSON.stringify(copy)); @@ -333,6 +384,9 @@ test('should report api step failure', async ({ runInlineTest }) => { test('should report test.step', async ({ runInlineTest }) => { const expectReporterJS = ` class Reporter { + onStdOut(chunk) { + process.stdout.write(chunk); + } onStepBegin(test, result, step) { const copy = { ...step, startTime: undefined, duration: undefined }; console.log('%%%% begin', JSON.stringify(copy)); diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.ts index e79ae29c05..f502520a6c 100644 --- a/tests/playwright-test/retry.spec.ts +++ b/tests/playwright-test/retry.spec.ts @@ -68,7 +68,7 @@ test('should retry timeout', async ({ runInlineTest }) => { await new Promise(f => setTimeout(f, 10000)); }); ` - }, { timeout: 100, retries: 2 }); + }, { timeout: 1000, retries: 2 }); expect(exitCode).toBe(1); expect(passed).toBe(0); expect(failed).toBe(1); diff --git a/tests/playwright-test/test-output-dir.spec.ts b/tests/playwright-test/test-output-dir.spec.ts index b49edb0416..dfdb4366d8 100644 --- a/tests/playwright-test/test-output-dir.spec.ts +++ b/tests/playwright-test/test-output-dir.spec.ts @@ -88,6 +88,7 @@ test('should include the project name', async ({ runInlineTest }) => { 'helper.ts': ` export const test = pwt.test.extend({ auto: [ async ({}, run, testInfo) => { + testInfo.snapshotSuffix = ''; await run(); }, { auto: true } ] }); diff --git a/types/test.d.ts b/types/test.d.ts index 697d941555..4b09f5d5c3 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -2297,6 +2297,35 @@ export interface PlaywrightWorkerOptions { * [fixtures.channel](https://playwright.dev/docs/api/class-fixtures#fixtures-channel) take priority over this. */ launchOptions: LaunchOptions; + /** + * Whether to automatically capture a screenshot after each test. Defaults to `'off'`. + * - `'off'`: Do not capture screenshots. + * - `'on'`: Capture screenshot after each test. + * - `'only-on-failure'`: Capture screenshot after each test failure. + * + * Learn more about [automatic screenshots](https://playwright.dev/docs/test-configuration#automatic-screenshots). + */ + screenshot: 'off' | 'on' | 'only-on-failure'; + /** + * Whether to record a trace for each test. Defaults to `'off'`. + * - `'off'`: Do not record a trace. + * - `'on'`: Record a trace for each test. + * - `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs. + * - `'on-first-retry'`: Record a trace only when retrying a test for the first time. + * + * Learn more about [recording trace](https://playwright.dev/docs/test-configuration#record-test-trace). + */ + trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace'; + /** + * Whether to record video for each test. Defaults to `'off'`. + * - `'off'`: Do not record video. + * - `'on'`: Record video for each test. + * - `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs. + * - `'on-first-retry'`: Record video only when retrying a test for the first time. + * + * Learn more about [recording video](https://playwright.dev/docs/test-configuration#record-video). + */ + video: VideoMode | { mode: VideoMode, size: ViewportSize }; } export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-video'; @@ -2390,35 +2419,6 @@ export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | * */ export interface PlaywrightTestOptions { - /** - * Whether to automatically capture a screenshot after each test. Defaults to `'off'`. - * - `'off'`: Do not capture screenshots. - * - `'on'`: Capture screenshot after each test. - * - `'only-on-failure'`: Capture screenshot after each test failure. - * - * Learn more about [automatic screenshots](https://playwright.dev/docs/test-configuration#automatic-screenshots). - */ - screenshot: 'off' | 'on' | 'only-on-failure'; - /** - * Whether to record a trace for each test. Defaults to `'off'`. - * - `'off'`: Do not record a trace. - * - `'on'`: Record a trace for each test. - * - `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs. - * - `'on-first-retry'`: Record a trace only when retrying a test for the first time. - * - * Learn more about [recording trace](https://playwright.dev/docs/test-configuration#record-test-trace). - */ - trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace'; - /** - * Whether to record video for each test. Defaults to `'off'`. - * - `'off'`: Do not record video. - * - `'on'`: Record video for each test. - * - `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs. - * - `'on-first-retry'`: Record video only when retrying a test for the first time. - * - * Learn more about [recording video](https://playwright.dev/docs/test-configuration#record-video). - */ - video: VideoMode | { mode: VideoMode, size: ViewportSize }; /** * Whether to automatically download all the attachments. Defaults to `false` where all the downloads are canceled. */ diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 534c6c63f1..c3ef4285de 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -284,14 +284,14 @@ export interface PlaywrightWorkerOptions { headless: boolean | undefined; channel: BrowserChannel | undefined; launchOptions: LaunchOptions; + screenshot: 'off' | 'on' | 'only-on-failure'; + trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace'; + video: VideoMode | { mode: VideoMode, size: ViewportSize }; } export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-video'; export interface PlaywrightTestOptions { - screenshot: 'off' | 'on' | 'only-on-failure'; - trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace'; - video: VideoMode | { mode: VideoMode, size: ViewportSize }; acceptDownloads: boolean | undefined; bypassCSP: boolean | undefined; colorScheme: ColorScheme | undefined;