diff --git a/docs/src/test-assertions-js.md b/docs/src/test-assertions-js.md index 169f74e248..fb1c0fe774 100644 --- a/docs/src/test-assertions-js.md +++ b/docs/src/test-assertions-js.md @@ -96,6 +96,25 @@ The same works with soft assertions: expect.soft(value, 'my soft assertion').toBe(56); ``` +## Polling + +You can convert any synchronous `expect` to an asynchronous polling one using `expect.poll`. + +The following method will poll given function until it returns HTTP status 200: + +```js +expect.poll(async () => { + const response = await page.request.get('https://api.example.com'); + return response.status(); +}, { + // Custom error message + message: 'make sure API eventually succeeds', // custom error message + // Poll for 10 seconds; defaults to 5 seconds. Pass 0 to disable timeout. + timeout: 10000, +}).toBe(200); +``` + + ## API reference See the following pages for Playwright-specific assertions: - [APIResponseAssertions] assertions for [APIResponse] diff --git a/packages/playwright-test/src/expect.ts b/packages/playwright-test/src/expect.ts index d053c4c01a..f640e17c96 100644 --- a/packages/playwright-test/src/expect.ts +++ b/packages/playwright-test/src/expect.ts @@ -15,6 +15,7 @@ */ import expectLibrary from 'expect'; +import { raceAgainstTimeout } from 'playwright-core/lib/utils/async'; import path from 'path'; import { INVERTED_COLOR, @@ -47,7 +48,8 @@ import { toMatchSnapshot, toHaveScreenshot, getSnapshotName } from './matchers/t import type { Expect, TestError } from './types'; import matchers from 'expect/build/matchers'; import { currentTestInfo } from './globals'; -import { serializeError, captureStackTrace } from './util'; +import { serializeError, captureStackTrace, currentExpectTimeout } from './util'; +import { monotonicTime } from 'playwright-core/lib/utils/utils'; // #region // Mirrored from https://github.com/facebook/jest/blob/f13abff8df9a0e1148baf3584bcde6d1b479edc7/packages/expect/src/print.ts @@ -89,21 +91,25 @@ export const printReceivedStringContainExpectedResult = ( // #endregion -function createExpect(actual: unknown, message: string|undefined, isSoft: boolean) { - if (message !== undefined && typeof message !== 'string') - throw new Error('expect(actual, optionalErrorMessage): optional error message must be a string.'); - return new Proxy(expectLibrary(actual), new ExpectMetaInfoProxyHandler(message || '', isSoft)); +type ExpectMessageOrOptions = undefined | string | { message?: string, timeout?: number }; + +function createExpect(actual: unknown, messageOrOptions: ExpectMessageOrOptions, isSoft: boolean, isPoll: boolean) { + return new Proxy(expectLibrary(actual), new ExpectMetaInfoProxyHandler(messageOrOptions, isSoft, isPoll)); } export const expect: Expect = new Proxy(expectLibrary as any, { - apply: function(target: any, thisArg: any, argumentsList: [actual: unknown, message: string|undefined]) { - const [actual, message] = argumentsList; - return createExpect(actual, message, false /* isSoft */); + apply: function(target: any, thisArg: any, argumentsList: [actual: unknown, messageOrOptions: ExpectMessageOrOptions]) { + const [actual, messageOrOptions] = argumentsList; + return createExpect(actual, messageOrOptions, false /* isSoft */, false /* isPoll */); } }); -expect.soft = (actual: unknown, message: string|undefined) => { - return createExpect(actual, message, true /* isSoft */); +expect.soft = (actual: unknown, messageOrOptions: ExpectMessageOrOptions) => { + return createExpect(actual, messageOrOptions, true /* isSoft */, false /* isPoll */); +}; + +expect.poll = (actual: unknown, messageOrOptions: ExpectMessageOrOptions) => { + return createExpect(actual, messageOrOptions, false /* isSoft */, true /* isPoll */); }; expectLibrary.setState({ expand: false }); @@ -133,19 +139,25 @@ const customMatchers = { }; type ExpectMetaInfo = { - message: string; + message?: string; isSoft: boolean; + isPoll: boolean; + pollTimeout?: number; }; let expectCallMetaInfo: undefined|ExpectMetaInfo = undefined; class ExpectMetaInfoProxyHandler { - private _message: string; - private _isSoft: boolean; + private _info: ExpectMetaInfo; - constructor(message: string, isSoft: boolean) { - this._message = message; - this._isSoft = isSoft; + constructor(messageOrOptions: ExpectMessageOrOptions, isSoft: boolean, isPoll: boolean) { + this._info = { isSoft, isPoll }; + if (typeof messageOrOptions === 'string') { + this._info.message = messageOrOptions; + } else { + this._info.message = messageOrOptions?.message; + this._info.pollTimeout = messageOrOptions?.timeout; + } } get(target: any, prop: any, receiver: any): any { @@ -157,15 +169,17 @@ class ExpectMetaInfoProxyHandler { if (!testInfo) return value.call(target, ...args); const handleError = (e: Error) => { - if (this._isSoft) + if (this._info.isSoft) testInfo._failWithError(serializeError(e), false /* isHardError */); else throw e; }; try { expectCallMetaInfo = { - message: this._message, - isSoft: this._isSoft, + message: this._info.message, + isSoft: this._info.isSoft, + isPoll: this._info.isPoll, + pollTimeout: this._info.pollTimeout, }; let result = value.call(target, ...args); if ((result instanceof Promise)) @@ -180,6 +194,36 @@ class ExpectMetaInfoProxyHandler { } } +async function pollMatcher(matcher: any, timeout: number, thisArg: any, generator: () => any, ...args: any[]) { + let result: { pass: boolean, message: () => string } | undefined = undefined; + const startTime = monotonicTime(); + const pollIntervals = [100, 250, 500]; + while (true) { + const elapsed = monotonicTime() - startTime; + if (timeout !== 0 && elapsed > timeout) + break; + const received = timeout !== 0 ? await raceAgainstTimeout(generator, timeout - elapsed) : await generator(); + if (received.timedOut) + break; + result = matcher.call(thisArg, received.result, ...args); + const success = result!.pass !== thisArg.isNot; + if (success) + return result; + await new Promise(x => setTimeout(x, pollIntervals.shift() ?? 1000)); + } + const timeoutMessage = `Timeout ${timeout}ms exceeded while waiting on the predicate`; + const message = result ? [ + result.message(), + '', + `Call Log:`, + `- ${timeoutMessage}`, + ].join('\n') : timeoutMessage; + return { + pass: thisArg.isNot, + message: () => message, + }; +} + function wrap(matcherName: string, matcher: any) { return function(this: any, ...args: any[]) { const testInfo = currentTestInfo(); @@ -196,10 +240,12 @@ function wrap(matcherName: string, matcher: any) { const frame = stackTrace.frames[0]; const customMessage = expectCallMetaInfo?.message ?? ''; const isSoft = expectCallMetaInfo?.isSoft ?? false; + const isPoll = expectCallMetaInfo?.isPoll ?? false; + const pollTimeout = expectCallMetaInfo?.pollTimeout; const step = testInfo._addStep({ location: frame && frame.file ? { file: path.resolve(process.cwd(), frame.file), line: frame.line || 0, column: frame.column || 0 } : undefined, category: 'expect', - title: customMessage || `expect${isSoft ? '.soft' : ''}${this.isNot ? '.not' : ''}.${matcherName}${titleSuffix}`, + title: customMessage || `expect${isPoll ? '.poll' : ''}${isSoft ? '.soft' : ''}${this.isNot ? '.not' : ''}.${matcherName}${titleSuffix}`, canHaveChildren: true, forceNoParent: false }); @@ -240,7 +286,19 @@ function wrap(matcherName: string, matcher: any) { }; try { - const result = matcher.call(this, ...args); + let result; + const [receivedOrGenerator, ...otherArgs] = args; + if (isPoll) { + if (typeof receivedOrGenerator !== 'function') + throw new Error('`expect.poll()` accepts only function as a first argument'); + if ((customMatchers as any)[matcherName] || matcherName === 'resolves' || matcherName === 'rejects') + throw new Error(`\`expect.poll()\` does not support "${matcherName}" matcher.`); + result = pollMatcher(matcher, currentExpectTimeout({ timeout: pollTimeout }), this, receivedOrGenerator, ...otherArgs); + } else { + if (typeof receivedOrGenerator === 'function') + throw new Error('Cannot accept function as a first argument; did you mean to use `expect.poll()`?'); + result = matcher.call(this, ...args); + } if (result instanceof Promise) return result.then(reportStepEnd).catch(reportStepError); return reportStepEnd(result); diff --git a/packages/playwright-test/types/testExpect.d.ts b/packages/playwright-test/types/testExpect.d.ts index fa6a6628c3..4978c271b6 100644 --- a/packages/playwright-test/types/testExpect.d.ts +++ b/packages/playwright-test/types/testExpect.d.ts @@ -28,8 +28,9 @@ type MakeMatchers = PlaywrightTest.Matchers & ExtraMatchers export declare type Expect = { - (actual: T, message?: string): MakeMatchers; - soft: (actual: T, message?: string) => MakeMatchers; + (actual: T, messageOrOptions?: string | { message?: string }): MakeMatchers; + soft: (actual: T, messageOrOptions?: string | { message?: string }) => MakeMatchers; + poll: (actual: () => T | Promise, messageOrOptions?: string | { message?: string, timeout?: number }) => Omit, 'rejects' | 'resolves'>; extend(arg0: any): void; getState(): expect.MatcherState; diff --git a/tests/playwright-test/expect-poll.spec.ts b/tests/playwright-test/expect-poll.spec.ts new file mode 100644 index 0000000000..91cd50b32d --- /dev/null +++ b/tests/playwright-test/expect-poll.spec.ts @@ -0,0 +1,146 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect, stripAnsi } from './playwright-test-fixtures'; + +test('should poll predicate', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should poll sync predicate', async () => { + let i = 0; + await test.expect.poll(() => ++i).toBe(3); + }); + test('should poll async predicate', async () => { + let i = 0; + await test.expect.poll(async () => { + await new Promise(x => setTimeout(x, 50)); + return ++i; + }).toBe(3); + }); + test('should poll predicate that returns a promise', async () => { + let i = 0; + await test.expect.poll(() => Promise.resolve(++i)).toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); +}); + +test('should compile', async ({ runTSC }) => { + const result = await runTSC({ + 'a.spec.ts': ` + const { test } = pwt; + test('should poll sync predicate', () => { + let i = 0; + test.expect.poll(() => ++i).toBe(3); + test.expect.poll(() => ++i, 'message').toBe(3); + test.expect.poll(() => ++i, { message: 'message' }).toBe(3); + test.expect.poll(() => ++i, { timeout: 100 }).toBe(3); + test.expect.poll(() => ++i, { message: 'message', timeout: 100 }).toBe(3); + test.expect.poll(async () => { + await new Promise(x => setTimeout(x, 50)); + return ++i; + }).toBe(3); + test.expect.poll(() => Promise.resolve(++i)).toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('should respect timeout', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail', async () => { + await test.expect.poll(() => false, { timeout: 100 }).toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('Timeout 100ms exceeded while waiting on the predicate'); + expect(stripAnsi(result.output)).toContain(` + 7 | await test.expect.poll(() => false, { timeout: 100 }). + `.trim()); +}); + +test('should fail when passed in non-function', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail', async () => { + await test.expect.poll(false).toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('Error: `expect.poll()` accepts only function as a first argument'); +}); + +test('should fail when used with web-first assertion', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail', async ({ page }) => { + await test.expect.poll(() => page.locator('body')).toHaveText('foo'); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('Error: `expect.poll()` does not support "toHaveText" matcher'); +}); + +test('should time out when running infinite predicate', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail', async ({ page }) => { + await test.expect.poll(() => new Promise(x => {}), { timeout: 100 }).toBe(42); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('Timeout 100ms exceeded'); +}); + +test('should show error that is thrown from predicate', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail', async ({ page }) => { + await test.expect.poll(() => { throw new Error('foo bar baz'); }, { timeout: 100 }).toBe(42); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('foo bar baz'); +}); + +test('should support .not predicate', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should fail', async ({ page }) => { + let i = 0; + await test.expect.poll(() => ++i).not.toBeLessThan(3); + expect(i).toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); diff --git a/tests/playwright-test/expect-soft.spec.ts b/tests/playwright-test/expect-soft.spec.ts index 29af49f18a..7002086841 100644 --- a/tests/playwright-test/expect-soft.spec.ts +++ b/tests/playwright-test/expect-soft.spec.ts @@ -23,6 +23,7 @@ test('soft expects should compile', async ({ runTSC }) => { test('should work', () => { test.expect.soft(1+1).toBe(3); test.expect.soft(1+1, 'custom error message').toBe(3); + test.expect.soft(1+1, { message: 'custom error message' }).toBe(3); }); ` }); @@ -43,6 +44,19 @@ test('soft expects should work', async ({ runInlineTest }) => { expect(stripAnsi(result.output)).toContain('woof-woof'); }); +test('should fail when passed in function', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('should work', () => { + test.expect.soft(() => 1+1).toBe(2); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain('Cannot accept function as a first argument; did you mean to use `expect.poll()`?'); +}); + test('should report a mixture of soft and non-soft errors', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.spec.ts': ` @@ -51,7 +65,7 @@ test('should report a mixture of soft and non-soft errors', async ({ runInlineTe test.expect.soft(1+1, 'one plus one').toBe(3); test.expect.soft(2*2, 'two times two').toBe(5); test.expect(3/3, 'three div three').toBe(7); - test.expect.soft(6-4, 'six minus four').toBe(3); + test.expect.soft(6-4, { message: 'six minus four' }).toBe(3); }); ` }); diff --git a/tests/playwright-test/expect.spec.ts b/tests/playwright-test/expect.spec.ts index bfb81a4634..e3fd55b60c 100644 --- a/tests/playwright-test/expect.spec.ts +++ b/tests/playwright-test/expect.spec.ts @@ -67,20 +67,6 @@ test('should not expand huge arrays', async ({ runInlineTest }) => { expect(result.output.length).toBeLessThan(100000); }); -test('should fail when passed `null` instead of message', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'expect-test.spec.ts': ` - const { test } = pwt; - test('custom expect message', () => { - test.expect(1+1, null).toEqual(3); - }); - ` - }); - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expect(stripAnsi(result.output)).toContain(`optional error message must be a string.`); -}); - test('should include custom error message', async ({ runInlineTest }) => { const result = await runInlineTest({ 'expect-test.spec.ts': ` @@ -105,7 +91,7 @@ test('should include custom error message with web-first assertions', async ({ r 'expect-test.spec.ts': ` const { test } = pwt; test('custom expect message', async ({page}) => { - await expect(page.locator('x-foo'), 'x-foo must be visible').toBeVisible({timeout: 1}); + await expect(page.locator('x-foo'), { message: 'x-foo must be visible' }).toBeVisible({timeout: 1}); }); ` });