Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore: land experimental tools (#34503)

Browse source
This commit is contained in:
Pavel Feldman 2025-01-27 14:49:38 -08:00 committed by GitHub
parent eaaef29dbd
commit bd2fdd0f20
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 1362 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

295
package-lock.json generated
View file

@ -111,6 +111,22 @@
"node": ">=6.0.0"
}
},
"node_modules/@anthropic-ai/sdk": {
"version": "0.33.1",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.33.1.tgz",
"integrity": "sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "^18.11.18",
"@types/node-fetch": "^2.6.4",
"abort-controller": "^3.0.0",
"agentkeepalive": "^4.2.1",
"form-data-encoder": "1.7.2",
"formdata-node": "^4.3.2",
"node-fetch": "^2.6.7"
}
},
"node_modules/@babel/cli": {
"version": "7.23.4",
"resolved": "https://registry.npmjs.org/@babel/cli/-/cli-7.23.4.tgz",
@ -1503,6 +1519,10 @@
"resolved": "packages/playwright-ct-vue",
"link": true
},
"node_modules/@playwright/experimental-tools": {
"resolved": "packages/playwright-tools",
"link": true
},
"node_modules/@playwright/test": {
"resolved": "packages/playwright-test",
"link": true
@ -1867,6 +1887,17 @@
"undici-types": "~5.26.4"
}
},
"node_modules/@types/node-fetch": {
"version": "2.6.12",
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz",
"integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*",
"form-data": "^4.0.0"
}
},
"node_modules/@types/prop-types": {
"version": "15.7.11",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz",
@ -2331,6 +2362,19 @@
"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
"dev": true
},
"node_modules/abort-controller": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
"dev": true,
"license": "MIT",
"dependencies": {
"event-target-shim": "^5.0.0"
},
"engines": {
"node": ">=6.5"
}
},
"node_modules/acorn": {
"version": "8.11.3",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
@ -2351,6 +2395,19 @@
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
}
},
"node_modules/agentkeepalive": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz",
"integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"humanize-ms": "^1.2.1"
},
"engines": {
"node": ">= 8.0.0"
}
},
"node_modules/ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
@ -2590,6 +2647,13 @@
"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
"dev": true
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
"dev": true,
"license": "MIT"
},
"node_modules/available-typed-arrays": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz",
@ -2898,6 +2962,19 @@
"node": ">=0.1.90"
}
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"dev": true,
"license": "MIT",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/commander": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
@ -3219,6 +3296,16 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/dequal": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
@ -3930,6 +4017,16 @@
"node": ">=0.10.0"
}
},
"node_modules/event-target-shim": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/extract-zip": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
@ -4077,6 +4174,42 @@
"is-callable": "^1.1.3"
}
},
"node_modules/form-data": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
"integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
"dev": true,
"license": "MIT",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/form-data-encoder": {
"version": "1.7.2",
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
"integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
"dev": true,
"license": "MIT"
},
"node_modules/formdata-node": {
"version": "4.4.1",
"resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
"integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"node-domexception": "1.0.0",
"web-streams-polyfill": "4.0.0-beta.3"
},
"engines": {
"node": ">= 12.20"
}
},
"node_modules/formidable": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
@ -4562,6 +4695,16 @@
"node": ">=10.19.0"
}
},
"node_modules/humanize-ms": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.0.0"
}
},
"node_modules/ignore": {
"version": "5.3.1",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",
@ -5358,6 +5501,29 @@
"node": ">=10.0.0"
}
},
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dev": true,
"license": "MIT",
"dependencies": {
"mime-db": "1.52.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mimic-response": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
@ -5438,6 +5604,47 @@
"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
"dev": true
},
"node_modules/node-domexception": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
"dev": true,
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "github",
"url": "https://paypal.me/jimmywarting"
}
],
"license": "MIT",
"engines": {
"node": ">=10.5.0"
}
},
"node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"dev": true,
"license": "MIT",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/node-releases": {
"version": "2.0.14",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz",
@ -5637,6 +5844,37 @@
"wrappy": "1"
}
},
"node_modules/openai": {
"version": "4.79.1",
"resolved": "https://registry.npmjs.org/openai/-/openai-4.79.1.tgz",
"integrity": "sha512-M7P5/PKnT/S/B5v0D64giC9mjyxFYkqlCuQFzR5hkdzMdqUuHf8T1gHhPGPF5oAvu4+PO3TvJv/qhZoS2bqAkw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@types/node": "^18.11.18",
"@types/node-fetch": "^2.6.4",
"abort-controller": "^3.0.0",
"agentkeepalive": "^4.2.1",
"form-data-encoder": "1.7.2",
"formdata-node": "^4.3.2",
"node-fetch": "^2.6.7"
},
"bin": {
"openai": "bin/cli"
},
"peerDependencies": {
"ws": "^8.18.0",
"zod": "^3.23.8"
},
"peerDependenciesMeta": {
"ws": {
"optional": true
},
"zod": {
"optional": true
}
}
},
"node_modules/optionator": {
"version": "0.9.3",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
@ -6754,6 +6992,13 @@
"node": ">=8.0"
}
},
"node_modules/tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
"dev": true,
"license": "MIT"
},
"node_modules/trace-viewer": {
"resolved": "packages/trace-viewer",
"link": true
@ -7492,6 +7737,34 @@
"resolved": "packages/web",
"link": true
},
"node_modules/web-streams-polyfill": {
"version": "4.0.0-beta.3",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
"integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 14"
}
},
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
"dev": true,
"license": "BSD-2-Clause"
},
"node_modules/whatwg-url": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
"dev": true,
"license": "MIT",
"dependencies": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"
}
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@ -7610,10 +7883,11 @@
"dev": true
},
"node_modules/ws": {
"version": "8.17.1",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
"integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
"integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=10.0.0"
},
@ -7932,6 +8206,21 @@
"node": ">=18"
}
},
"packages/playwright-tools": {
"name": "@playwright/experimental-tools",
"version": "1.51.0-next",
"license": "Apache-2.0",
"dependencies": {
"playwright": "1.51.0-next"
},
"devDependencies": {
"@anthropic-ai/sdk": "^0.33.1",
"openai": "^4.79.1"
},
"engines": {
"node": ">=18"
}
},
"packages/playwright-webkit": {
"version": "1.51.0-next",
"hasInstallScript": true,

5
packages/playwright-core/src/client/elementHandle.ts
View file

@ -62,6 +62,11 @@ export class ElementHandle<T extends Node = Node> extends JSHandle<T> implements
return Frame.fromNullable((await this._elementChannel.contentFrame()).frame);
}
async _generateLocatorString(): Promise<string | null> {
const value = (await this._elementChannel.generateLocatorString()).value;
return value === undefined ? null : value;
}
async getAttribute(name: string): Promise<string | null> {
const value = (await this._elementChannel.getAttribute({ name })).value;
return value === undefined ? null : value;

8
packages/playwright-core/src/client/locator.ts
View file

@ -236,6 +236,10 @@ export class Locator implements api.Locator {
return await this._frame._queryCount(this._selector);
}
async _generateLocatorString(): Promise<string | null> {
return await this._withElement(h => h._generateLocatorString());
}
async getAttribute(name: string, options?: TimeoutOptions): Promise<string | null> {
return await this._frame.getAttribute(this._selector, name, { strict: true, ...options });
}
@ -288,8 +292,8 @@ export class Locator implements api.Locator {
return await this._withElement((h, timeout) => h.screenshot({ ...options, timeout }), options.timeout);
}
async ariaSnapshot(options?: TimeoutOptions): Promise<string> {
const result = await this._frame._channel.ariaSnapshot({ ...options, selector: this._selector });
async ariaSnapshot(options?: { _id?: boolean, _mode?: 'raw' | 'regex' } & TimeoutOptions): Promise<string> {
const result = await this._frame._channel.ariaSnapshot({ ...options, id: options?._id, mode: options?._mode, selector: this._selector });
return result.snapshot;
}

6
packages/playwright-core/src/protocol/validator.ts
View file

@ -1429,6 +1429,8 @@ scheme.FrameAddStyleTagResult = tObject({
});
scheme.FrameAriaSnapshotParams = tObject({
selector: tString,
id: tOptional(tBoolean),
mode: tOptional(tEnum(['raw', 'regex'])),
timeout: tOptional(tNumber),
});
scheme.FrameAriaSnapshotResult = tObject({
@ -1925,6 +1927,10 @@ scheme.ElementHandleFillParams = tObject({
scheme.ElementHandleFillResult = tOptional(tObject({}));
scheme.ElementHandleFocusParams = tOptional(tObject({}));
scheme.ElementHandleFocusResult = tOptional(tObject({}));
scheme.ElementHandleGenerateLocatorStringParams = tOptional(tObject({}));
scheme.ElementHandleGenerateLocatorStringResult = tObject({
value: tOptional(tString),
});
scheme.ElementHandleGetAttributeParams = tObject({
name: tString,
});

4
packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts
View file

@ -63,6 +63,10 @@ export class ElementHandleDispatcher extends JSHandleDispatcher implements chann
return { frame: frame ? FrameDispatcher.from(this._browserContextDispatcher(), frame) : undefined };
}
async generateLocatorString(params: channels.ElementHandleGenerateLocatorStringParams, metadata: CallMetadata): Promise<channels.ElementHandleGenerateLocatorStringResult> {
return { value: await this._elementHandle.generateLocatorString() };
}
async getAttribute(params: channels.ElementHandleGetAttributeParams, metadata: CallMetadata): Promise<channels.ElementHandleGetAttributeResult> {
const value = await this._elementHandle.getAttribute(metadata, params.name);
return { value: value === null ? undefined : value };

15
packages/playwright-core/src/server/dom.ts
View file

@ -28,7 +28,7 @@ import type { Progress } from './progress';
import { ProgressController } from './progress';
import type * as types from './types';
import type { TimeoutOptions } from '../common/types';
import { isUnderTest } from '../utils';
import { asLocator, isUnderTest } from '../utils';
import { prepareFilesForUpload } from './fileUploadUtils';
export type InputFilesItems = {
@ -185,6 +185,15 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
return this._page._delegate.getContentFrame(this);
}
async generateLocatorString(): Promise<string | undefined> {
const selector = await this.evaluateInUtility(async ([injected, node]) => {
return injected.generateSelectorSimple(node as unknown as Element);
}, {});
if (selector === 'error:notconnected')
return;
return asLocator('javascript', selector);
}
async getAttribute(metadata: CallMetadata, name: string): Promise<string | null> {
return this._frame.getAttribute(metadata, ':scope', name, {}, this);
}
@ -799,8 +808,8 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
return this._page._delegate.getBoundingBox(this);
}
async ariaSnapshot(): Promise<string> {
return await this.evaluateInUtility(([injected, element]) => injected.ariaSnapshot(element), {});
async ariaSnapshot(options: { id?: boolean, mode?: 'raw' | 'regex' }): Promise<string> {
return await this.evaluateInUtility(([injected, element, options]) => injected.ariaSnapshot(element, options), options);
}
async screenshot(metadata: CallMetadata, options: ScreenshotOptions & TimeoutOptions = {}): Promise<Buffer> {

4
packages/playwright-core/src/server/frames.ts
View file

@ -1408,10 +1408,10 @@ export class Frame extends SdkObject {
});
}
async ariaSnapshot(metadata: CallMetadata, selector: string, options: types.TimeoutOptions = {}): Promise<string> {
async ariaSnapshot(metadata: CallMetadata, selector: string, options: { id?: boolean, mode?: 'raw' | 'regex' } & types.TimeoutOptions = {}): Promise<string> {
const controller = new ProgressController(metadata, this);
return controller.run(async progress => {
return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot());
return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot(options));
}, this._page._timeoutSettings.timeout(options));
}

6
packages/playwright-core/src/server/injected/consoleApi.ts
View file

@ -85,10 +85,8 @@ class ConsoleAPI {
inspect: (selector: string) => this._inspect(selector),
selector: (element: Element) => this._selector(element),
generateLocator: (element: Element, language?: Language) => this._generateLocator(element, language),
ariaSnapshot: (element?: Element) => {
const snapshot = this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body);
// eslint-disable-next-line no-console
console.log(snapshot);
ariaSnapshot: (element?: Element, options?: { id?: boolean }) => {
return this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body, options);
},
resume: () => this._resume(),
...new Locator(injectedScript, ''),

14
packages/playwright-core/src/server/injected/injectedScript.ts
View file

@ -72,6 +72,7 @@ export class InjectedScript {
// eslint-disable-next-line no-restricted-globals
readonly window: Window & typeof globalThis;
readonly document: Document;
private _ariaElementById: Map<number, Element> | undefined;
// Recorder must use any external dependencies through InjectedScript.
// Otherwise it will end up with a copy of all modules it uses, and any
@ -130,6 +131,7 @@ export class InjectedScript {
this._engines.set('internal:attr', this._createNamedAttributeEngine());
this._engines.set('internal:testid', this._createNamedAttributeEngine());
this._engines.set('internal:role', createRoleEngine(true));
this._engines.set('internal:aria-id', this._createAriaIdEngine());
for (const { name, engine } of customEngines)
this._engines.set(name, engine);
@ -221,7 +223,8 @@ export class InjectedScript {
if (node.nodeType !== Node.ELEMENT_NODE)
throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');
const ariaSnapshot = generateAriaTree(node as Element);
return renderAriaTree(ariaSnapshot.root, options);
this._ariaElementById = ariaSnapshot.elements;
return renderAriaTree(ariaSnapshot.root, { ...options, ids: options?.id ? ariaSnapshot.ids : undefined });
}
ariaSnapshotAsObject(node: Node): AriaSnapshot {
@ -609,6 +612,15 @@ export class InjectedScript {
return result;
}
_createAriaIdEngine() {
const queryAll = (root: SelectorRoot, selector: string): Element[] => {
const ariaId = parseInt(selector, 10);
const result = this._ariaElementById?.get(ariaId);
return result && result.isConnected ? [result] : [];
};
return { queryAll };
}
elementState(node: Node, state: ElementStateWithoutStable): ElementStateQueryResult {
const element = this.retarget(node, ['visible', 'hidden'].includes(state) ? 'none' : 'follow-label');
if (!element || !element.isConnected) {

2
packages/playwright-core/src/server/selectors.ts
View file

@ -39,7 +39,7 @@ export class Selectors {
'internal:has', 'internal:has-not',
'internal:has-text', 'internal:has-not-text',
'internal:and', 'internal:or', 'internal:chain',
'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid',
'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid', 'internal:aria-id'
]);
this._builtinEnginesInMainWorld = new Set([
'_react', '_vue',

8
packages/playwright-tools/.npmignore Normal file
View file

@ -0,0 +1,8 @@
**/*
README.md
LICENSE
!lib/*
!browser.js
!browser.d.ts
!computer-20241022.js
!computer-20241022.d.ts

1
packages/playwright-tools/README.md Normal file
View file

@ -0,0 +1 @@
> **BEWARE** This package is EXPERIMENTAL and does not respect semver.

30
packages/playwright-tools/browser.d.ts vendored Normal file
View file

@ -0,0 +1,30 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type playwright from 'playwright';
import { ToolDeclaration, JSONSchemaType } from './types';
export type ToolResult = {
error?: string;
code: Array<string>;
snapshot: string;
}
export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;
export const schema: ToolDeclaration[];
export const call: ToolCall;
export const snapshot: (page) => Promise<string>;

19
packages/playwright-tools/browser.js Normal file
View file

@ -0,0 +1,19 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { schema, call, snapshot } = require('./lib/tools/browser');
module.exports = { schema, call, snapshot };

28
packages/playwright-tools/computer-20241022.d.ts vendored Normal file
View file

@ -0,0 +1,28 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type playwright from 'playwright';
import { JSONSchemaType } from './types';
export type ToolResult = {
output?: string;
error?: string;
base64_image?: string;
};
export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;
export const call: ToolCall;

19
packages/playwright-tools/computer-20241022.js Normal file
View file

@ -0,0 +1,19 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { call } = require('./lib/tools/computer-20241022');
module.exports = { call };

35
packages/playwright-tools/package.json Normal file
View file

@ -0,0 +1,35 @@
{
"name": "@playwright/experimental-tools",
"version": "1.51.0-next",
"description": "Playwright Tools for AI",
"repository": {
"type": "git",
"url": "git+https://github.com/microsoft/playwright.git"
},
"homepage": "https://playwright.dev",
"engines": {
"node": ">=18"
},
"author": {
"name": "Microsoft Corporation"
},
"license": "Apache-2.0",
"exports": {
"./browser": {
"types": "./browser.d.ts",
"default": "./browser.js"
},
"./computer-20241022": {
"types": "./computer-20241022.d.ts",
"default": "./computer-20241022.js"
},
"./package.json": "./package.json"
},
"dependencies": {
"playwright": "1.51.0-next"
},
"devDependencies": {
"@anthropic-ai/sdk": "^0.33.1",
"openai": "^4.79.1"
}
}

143
packages/playwright-tools/src/examples/browser-anthropic.ts Normal file
View file

@ -0,0 +1,143 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* eslint-disable no-console */
import playwright from 'playwright';
import Anthropic from '@anthropic-ai/sdk';
import dotenv from 'dotenv';
import browser from '@playwright/experimental-tools/browser';
dotenv.config();
const anthropic = new Anthropic();
export const system = `
You are a web tester.
<Instructions>
- Perform test according to the provided checklist
- Use browser tools to perform actions on web page
- Never ask questions, always perform a best guess action
- Use one tool at a time, wait for its result before proceeding.
- When ready use "reportResult" tool to report result
</Instructions>`;
const reportTool: Anthropic.Tool = {
name: 'reportResult',
description: 'Submit test result',
input_schema: {
type: 'object',
properties: {
'success': { type: 'boolean', description: 'Whether test passed' },
'result': { type: 'string', description: 'Result of the test if some information has been requested' },
'error': { type: 'string', description: 'Error message if test failed' }
},
required: ['success']
}
};
type Message = Anthropic.Beta.Messages.BetaMessageParam & {
history: Anthropic.Beta.Messages.BetaMessageParam['content']
};
async function anthropicAgentLoop(page: playwright.Page, task: string) {
// Convert them into tools for Anthropic.
const pageTools: Anthropic.Tool[] = browser.schema.map(tool => {
return {
name: tool.name,
description: tool.description,
input_schema: tool.parameters as any,
};
});
// Add report tool.
const tools = [reportTool, ...pageTools];
const history: Message[] = [{
role: 'user',
history: `Task: ${task}`,
content: `Task: ${task}\n\n${await browser.snapshot(page)}`,
}];
// Run agentic loop, cap steps.
for (let i = 0; i < 50; i++) {
const response = await anthropic.messages.create({
model: 'claude-3-5-sonnet-20241022',
max_tokens: 1024,
temperature: 0,
tools,
system,
messages: toAnthropicMessages(history),
});
history.push({ role: 'assistant', content: response.content, history: response.content });
const toolUse = response.content.find(block => block.type === 'tool_use');
if (!toolUse) {
history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });
continue;
}
if (toolUse.name === 'reportResult') {
console.log(toolUse.input);
return;
}
// Run the Playwright tool.
const { error, snapshot, code } = await browser.call(page, toolUse.name, toolUse.input as any);
if (code.length)
console.log(code.join('\n'));
// Report the result.
const resultText = error ? `Error: ${error}\n` : 'Done\n';
history.push({
role: 'user',
content: [{
type: 'tool_result',
tool_use_id: toolUse.id,
content: [{ type: 'text', text: resultText + snapshot }],
}],
history: [{
type: 'tool_result',
tool_use_id: toolUse.id,
content: [{ type: 'text', text: resultText }],
}],
});
}
}
function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] {
return messages.map((message, i) => {
if (i === messages.length - 1)
return { ...message, history: undefined };
return { ...message, content: message.history, history: undefined };
});
}
async function main() {
const browser = await playwright.chromium.launch({ headless: false });
const page = await browser.newPage();
await anthropicAgentLoop(page, `
- Go to http://github.com/microsoft
- Search for "playwright" repository
- Navigate to it
- Switch into the Issues tab
- Report 3 first issues
`);
await browser.close();
}
void main();

161
packages/playwright-tools/src/examples/browser-openai.ts Normal file
View file

@ -0,0 +1,161 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* eslint-disable no-console */
import playwright from 'playwright';
import browser from '@playwright/experimental-tools/browser';
import dotenv from 'dotenv';
import OpenAI from 'openai';
import type { ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources';
dotenv.config();
const openai = new OpenAI();
export const system = `
You are a web tester.
<Instructions>to
- Perform test according to the provided checklist
- Use browser tools to perform actions on web page
- Never ask questions, always perform a best guess action
- When ready use "reportResult" tool to report result
- You can only make one tool call at a time.
</Instructions>`;
type Message = ChatCompletionMessageParam & {
history: any
};
const reportTool: ChatCompletionTool = {
type: 'function',
function: {
name: 'reportResult',
description: 'Submit test result',
parameters: {
type: 'object',
properties: {
success: { type: 'boolean', description: 'Whether test passed' },
result: { type: 'string', description: 'Result of the test if requested' },
error: { type: 'string', description: 'Error if test failed' },
},
required: ['success'],
additionalProperties: false,
},
}
};
async function openAIAgentLoop(page: playwright.Page, task: string) {
const pageTools: ChatCompletionTool[] = browser.schema.map(tool => ({
type: 'function',
function: {
name: tool.name,
description: tool.description,
parameters: {
...tool.parameters,
additionalProperties: false,
},
}
}));
const tools = [reportTool, ...pageTools];
const history: Message[] = [
{
role: 'system', content: system, history: system
},
{
role: 'user',
history: `Task: ${task}`,
content: `Task: ${task}\n\n${await browser.snapshot(page)}`,
}
];
// Run agentic loop, cap steps.
for (let i = 0; i < 50; i++) {
const completion = await openai.chat.completions.create({
model: 'gpt-4o',
messages: toOpenAIMessages(history),
tools,
store: true,
});
console.log(JSON.stringify(completion, null, 2));
const toolCalls = completion.choices[0]?.message?.tool_calls;
if (!toolCalls || toolCalls.length !== 1 || toolCalls[0].type !== 'function') {
history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });
continue;
}
const toolCall = toolCalls[0];
if (toolCall.function.name === 'reportResult') {
console.log(JSON.parse(toolCall.function.arguments));
return;
}
history.push({ ...completion.choices[0].message, history: null });
// Run the Playwright tool.
const params = JSON.parse(toolCall.function.arguments);
const { error, snapshot, code } = await browser.call(page, toolCall.function.name, params);
console.log({ error, code, snapshot });
if (code.length)
console.log(code.join('\n'));
if (toolCall.function.name === 'log')
return;
// Report the result.
const resultText = error ? `Error: ${error}\n` : 'Done\n';
history.push({
role: 'tool',
tool_call_id: toolCall.id,
content: resultText + snapshot,
history: resultText,
});
}
}
function toOpenAIMessages(messages: Message[]): ChatCompletionMessageParam[] {
return messages.map((message, i) => {
const copy: Message = { ...message };
delete copy.history;
if (i === messages.length - 1)
return copy;
copy.content = message.history;
return copy;
});
}
async function main() {
const browser = await playwright.chromium.launch({ headless: false });
const page = await browser.newPage();
await openAIAgentLoop(page, `
- Go to http://github.com/microsoft
- Search for "playwright" repository
- Navigate to it
- Capture snapshot for toolbar with Code, Issues, etc.
- Capture snapshot for branch selector
- Assert that number of Issues is present
- Switch into the Issues tab
- Report 3 first issues
`);
await browser.close();
}
void main();

148
packages/playwright-tools/src/examples/computer-20241022-anthropic.ts Normal file
View file

@ -0,0 +1,148 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* eslint-disable no-console */
import playwright from 'playwright';
import Anthropic from '@anthropic-ai/sdk';
import dotenv from 'dotenv';
import computer, { type ToolResult } from '@playwright/experimental-tools/computer-20241022';
import type { BetaImageBlockParam, BetaTextBlockParam } from '@anthropic-ai/sdk/resources/beta/messages/messages';
dotenv.config();
const anthropic = new Anthropic();
export const system = `
You are a web tester.
<Instructions>
- Perform test according to the provided checklist
- Use browser tools to perform actions on web page
- Never ask questions, always perform a best guess action
- Use one tool at a time, wait for its result before proceeding.
- When ready use "reportResult" tool to report result
</Instructions>`;
const computerTool: Anthropic.Beta.BetaToolUnion = {
type: 'computer_20241022',
name: 'computer',
display_width_px: 1920,
display_height_px: 1080,
display_number: 1,
};
const reportTool: Anthropic.Tool = {
name: 'reportResult',
description: 'Submit test result',
input_schema: {
type: 'object',
properties: {
'success': { type: 'boolean', description: 'Whether test passed' },
'result': { type: 'string', description: 'Result of the test if some information has been requested' },
'error': { type: 'string', description: 'Error message if test failed' }
},
required: ['success']
}
};
type Message = Anthropic.Beta.Messages.BetaMessageParam & {
history: Anthropic.Beta.Messages.BetaMessageParam['content']
};
async function anthropicAgentLoop(page: playwright.Page, task: string) {
// Add report tool.
const tools = [reportTool, computerTool];
const history: Message[] = [{
role: 'user',
history: `Task: ${task}`,
content: `Task: ${task}`,
}];
// Run agentic loop, cap steps.
for (let i = 0; i < 50; i++) {
const response = await anthropic.beta.messages.create({
model: 'claude-3-5-sonnet-20241022',
max_tokens: 1024,
temperature: 0,
tools,
system,
messages: toAnthropicMessages(history),
betas: ['computer-use-2024-10-22'],
});
history.push({ role: 'assistant', content: response.content, history: response.content });
const toolUse = response.content.find(block => block.type === 'tool_use');
if (!toolUse) {
history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });
continue;
}
if (toolUse.name === 'reportResult') {
console.log(toolUse.input);
return;
}
const result: ToolResult = await computer.call(page, toolUse.name, toolUse.input as any);
const contentEntry: BetaTextBlockParam | BetaImageBlockParam = result.base64_image ? {
type: 'image',
source: { type: 'base64', media_type: 'image/jpeg', data: result.base64_image }
} : {
type: 'text',
text: result.output || '',
};
history.push({
role: 'user',
content: [{
type: 'tool_result',
tool_use_id: toolUse.id,
content: [contentEntry],
}],
history: [{
type: 'tool_result',
tool_use_id: toolUse.id,
content: [{ type: 'text', text: '<redacted>' }],
}],
});
}
}
function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] {
return messages.map((message, i) => {
if (i === messages.length - 1)
return { ...message, history: undefined };
return { ...message, content: message.history, history: undefined };
});
}
const githubTask = `
- Search for "playwright" repository
- Navigate to it
- Switch into the Issues tab
- Report 3 first issues
`;
async function main() {
const browser = await playwright.chromium.launch({ headless: false });
const page = await browser.newPage();
await page.goto('http://github.com/microsoft');
await anthropicAgentLoop(page, githubTask);
await browser.close();
}
void main();

146
packages/playwright-tools/src/tools/browser.ts Normal file
View file

@ -0,0 +1,146 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type playwright from 'playwright';
import type { JSONSchemaType, ToolDeclaration } from '../../types';
import type { ToolResult } from '../../browser';
import { waitForNetwork } from './utils';
type LocatorEx = playwright.Locator & {
_generateLocatorString: () => Promise<string>;
};
const intentProperty = {
intent: {
type: 'string',
description: 'Intent behind this particular action. Used as a comment.',
}
};
const elementIdProperty = {
elementId: {
type: 'number',
description: 'Target element',
}
};
export const schema: ToolDeclaration[] = [
{
name: 'navigate',
description: 'Navigate to a URL',
parameters: {
type: 'object',
properties: {
...intentProperty,
url: {
type: 'string',
description: 'URL to navigate to',
},
},
required: ['intent', 'elementId'],
}
},
{
name: 'click',
description: 'Perform click on a web page',
parameters: {
type: 'object',
properties: {
...intentProperty,
...elementIdProperty,
},
required: ['intent', 'elementId'],
}
},
{
name: 'enterText',
description: 'Enter text into editable element',
parameters: {
type: 'object',
properties: {
...intentProperty,
...elementIdProperty,
text: {
type: 'string',
description: 'Text to enter',
},
submit: {
type: 'boolean',
description: 'Whether to submit entered text (press Enter after)'
}
},
required: ['intent', 'elementId', 'text'],
}
},
{
name: 'wait',
description: `Wait for given amount of time to see if the page updates. Use it after action if you think page is not ready yet`,
parameters: {
type: 'object',
properties: {
...intentProperty,
time: {
type: 'integer',
description: 'Time to wait in seconds',
},
},
required: ['intent', 'time'],
}
},
];
export async function call(page: playwright.Page, toolName: string, params: Record<string, JSONSchemaType>): Promise<ToolResult> {
const code: string[] = [];
try {
await waitForNetwork(page, async () => {
await performAction(page, toolName, params, code);
});
} catch (e) {
return { error: e.message, snapshot: await snapshot(page), code };
}
return { snapshot: await snapshot(page), code };
}
export async function snapshot(page: playwright.Page) {
const params = { _id: true } as any;
return `<Page snapshot>\n${await page.locator('body').ariaSnapshot(params)}\n</Page snapshot>`;
}
async function performAction(page: playwright.Page, toolName: string, params: Record<string, JSONSchemaType>, code: string[]) {
const locator = elementLocator(page, params);
code.push((params.intent as string).split('\n').map(line => `// ${line}`).join('\n'));
if (toolName === 'navigate') {
code.push(`await page.goto(${JSON.stringify(params.url)})`);
await page.goto(params.url as string);
} else if (toolName === 'click') {
code.push(`await page.${await locator._generateLocatorString()}.click()`);
await locator.click();
} else if (toolName === 'enterText') {
code.push(`await page.${await locator._generateLocatorString()}.click()`);
await locator.click();
code.push(`await page.${await locator._generateLocatorString()}.fill(${JSON.stringify(params.text)})`);
await locator.fill(params.text as string);
if (params.submit) {
code.push(`await page.${await locator._generateLocatorString()}.press("Enter")`);
await locator.press('Enter');
}
}
}
function elementLocator(page: playwright.Page, params: any): LocatorEx {
return page.locator(`internal:aria-id=${params.elementId}`) as LocatorEx;
}

158
packages/playwright-tools/src/tools/computer-20241022.ts Normal file
View file

@ -0,0 +1,158 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the 'License');
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an 'AS IS' BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type playwright from 'playwright';
import type { JSONSchemaType } from '../../types';
import type { ToolResult } from '../../computer-20241022';
import { waitForNetwork } from './utils';
export async function call(page: playwright.Page, toolName: string, input: Record<string, JSONSchemaType>): Promise<ToolResult> {
if (toolName !== 'computer')
throw new Error('Unsupported tool');
return await waitForNetwork(page, async () => {
return await performAction(page, toolName, input);
});
}
type PageState = {
x: number;
y: number;
};
const pageStateSymbol = Symbol('pageState');
function pageState(page: playwright.Page): PageState {
if (!(page as any)[pageStateSymbol])
(page as any)[pageStateSymbol] = { x: 0, y: 0 };
return (page as any)[pageStateSymbol];
}
async function performAction(page: playwright.Page, toolName: string, input: Record<string, JSONSchemaType>): Promise<ToolResult> {
const state = pageState(page);
const { action } = input as { action: string };
if (action === 'screenshot') {
const screenshot = await page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' });
return {
output: 'Screenshot',
base64_image: screenshot.toString('base64'),
};
}
if (action === 'mouse_move') {
const { coordinate } = input as { coordinate: [number, number] };
state.x = coordinate[0];
state.y = coordinate[1];
await page.mouse.move(state.x, state.y);
return { output: 'Mouse moved' };
}
if (action === 'left_click') {
await page.mouse.down();
await page.mouse.up();
return { output: 'Left clicked' };
}
if (action === 'left_click_drag') {
await page.mouse.down();
const { coordinate } = input as { coordinate: [number, number] };
state.x = coordinate[0];
state.y = coordinate[1];
await page.mouse.move(state.x, state.y);
await page.mouse.up();
return { output: 'Left dragged' };
}
if (action === 'right_click') {
await page.mouse.down({ button: 'right' });
await page.mouse.up({ button: 'right' });
return { output: 'Right clicked' };
}
if (action === 'double_click') {
await page.mouse.down();
await page.mouse.up();
await page.mouse.down();
await page.mouse.up();
return { output: 'Double clicked' };
}
if (action === 'middle_click') {
await page.mouse.down({ button: 'middle' });
await page.mouse.up({ button: 'middle' });
return { output: 'Middle clicked' };
}
if (action === 'key') {
const { text } = input as { text: string };
await page.keyboard.press(xToPlaywright(text));
return { output: 'Text typed' };
}
if (action === 'cursor_position')
return { output: `X=${state.x},Y=${state.y}` };
throw new Error('Unimplemented tool: ' + toolName);
}
const xToPlaywrightKeyMap = new Map([
['BackSpace', 'Backspace'],
['Tab', 'Tab'],
['Return', 'Enter'],
['Escape', 'Escape'],
['space', ' '],
['Delete', 'Delete'],
['Home', 'Home'],
['End', 'End'],
['Left', 'ArrowLeft'],
['Up', 'ArrowUp'],
['Right', 'ArrowRight'],
['Down', 'ArrowDown'],
['Insert', 'Insert'],
['Page_Up', 'PageUp'],
['Page_Down', 'PageDown'],
['F1', 'F1'],
['F2', 'F2'],
['F3', 'F3'],
['F4', 'F4'],
['F5', 'F5'],
['F6', 'F6'],
['F7', 'F7'],
['F8', 'F8'],
['F9', 'F9'],
['F10', 'F10'],
['F11', 'F11'],
['F12', 'F12'],
['Shift_L', 'Shift'],
['Shift_R', 'Shift'],
['Control_L', 'Control'],
['Control_R', 'Control'],
['Alt_L', 'Alt'],
['Alt_R', 'Alt'],
['Super_L', 'Meta'],
['Super_R', 'Meta'],
]);
const xToPlaywrightModifierMap = new Map([
['alt', 'Alt'],
['control', 'Control'],
['meta', 'Meta'],
['shift', 'Shift'],
]);
const xToPlaywright = (key: string) => {
const tokens = key.split('+');
if (tokens.length === 1)
return xToPlaywrightKeyMap.get(key) || key;
if (tokens.length === 2) {
const modifier = xToPlaywrightModifierMap.get(tokens[0]);
const key = xToPlaywrightKeyMap.get(tokens[1]) || tokens[1];
return modifier + '+' + key;
}
throw new Error('Invalid key: ' + key);
};

71
packages/playwright-tools/src/tools/utils.ts Normal file
View file

@ -0,0 +1,71 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type playwright from 'playwright';
import { ManualPromise } from 'playwright-core/lib/utils';
export async function waitForNetwork<R>(page: playwright.Page, callback: () => Promise<R>): Promise<R> {
const requests = new Set<playwright.Request>();
let frameNavigated = false;
const waitBarrier = new ManualPromise();
const requestListener = (request: playwright.Request) => requests.add(request);
const requestFinishedListener = (request: playwright.Request) => {
requests.delete(request);
if (!requests.size)
waitBarrier.resolve();
};
const frameNavigateListener = (frame: playwright.Frame) => {
if (frame.parentFrame())
return;
frameNavigated = true;
dispose();
clearTimeout(timeout);
void frame.waitForLoadState('load').then(() => {
waitBarrier.resolve();
});
};
const onTimeout = () => {
dispose();
waitBarrier.resolve();
};
page.on('request', requestListener);
page.on('requestfinished', requestFinishedListener);
page.on('framenavigated', frameNavigateListener);
const timeout = setTimeout(onTimeout, 10000);
const dispose = () => {
page.off('request', requestListener);
page.off('requestfinished', requestFinishedListener);
page.off('framenavigated', frameNavigateListener);
clearTimeout(timeout);
};
try {
const result = await callback();
if (!requests.size && !frameNavigated)
waitBarrier.resolve();
await waitBarrier;
await page.evaluate(() => new Promise(f => setTimeout(f, 1000)));
return result;
} finally {
dispose();
}
}

27
packages/playwright-tools/types.d.ts vendored Normal file
View file

@ -0,0 +1,27 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type playwright from 'playwright';
export type JSONSchemaType = string | number | boolean | JSONSchemaObject | JSONSchemaArray | null;
interface JSONSchemaObject { [key: string]: JSONSchemaType; }
interface JSONSchemaArray extends Array<JSONSchemaType> {}
export type ToolDeclaration = {
name: string;
description: string;
parameters: any;
};

10
packages/protocol/src/channels.d.ts vendored
View file

@ -2619,9 +2619,13 @@ export type FrameAddStyleTagResult = {
};
export type FrameAriaSnapshotParams = {
selector: string,
id?: boolean,
mode?: 'raw' | 'regex',
timeout?: number,
};
export type FrameAriaSnapshotOptions = {
id?: boolean,
mode?: 'raw' | 'regex',
timeout?: number,
};
export type FrameAriaSnapshotResult = {
@ -3311,6 +3315,7 @@ export interface ElementHandleChannel extends ElementHandleEventTarget, JSHandle
dispatchEvent(params: ElementHandleDispatchEventParams, metadata?: CallMetadata): Promise<ElementHandleDispatchEventResult>;
fill(params: ElementHandleFillParams, metadata?: CallMetadata): Promise<ElementHandleFillResult>;
focus(params?: ElementHandleFocusParams, metadata?: CallMetadata): Promise<ElementHandleFocusResult>;
generateLocatorString(params?: ElementHandleGenerateLocatorStringParams, metadata?: CallMetadata): Promise<ElementHandleGenerateLocatorStringResult>;
getAttribute(params: ElementHandleGetAttributeParams, metadata?: CallMetadata): Promise<ElementHandleGetAttributeResult>;
hover(params: ElementHandleHoverParams, metadata?: CallMetadata): Promise<ElementHandleHoverResult>;
innerHTML(params?: ElementHandleInnerHTMLParams, metadata?: CallMetadata): Promise<ElementHandleInnerHTMLResult>;
@ -3450,6 +3455,11 @@ export type ElementHandleFillResult = void;
export type ElementHandleFocusParams = {};
export type ElementHandleFocusOptions = {};
export type ElementHandleFocusResult = void;
export type ElementHandleGenerateLocatorStringParams = {};
export type ElementHandleGenerateLocatorStringOptions = {};
export type ElementHandleGenerateLocatorStringResult = {
value?: string,
};
export type ElementHandleGetAttributeParams = {
name: string,
};

10
packages/protocol/src/protocol.yml
View file

@ -1879,6 +1879,12 @@ Frame:
ariaSnapshot:
parameters:
selector: string
id: boolean?
mode:
type: enum?
literals:
- raw
- regex
timeout: number?
returns:
snapshot: string
@ -2648,6 +2654,10 @@ ElementHandle:
slowMo: true
snapshot: true
generateLocatorString:
returns:
value: string?
getAttribute:
parameters:
name: string

5
utils/workspace.js
View file

@ -167,6 +167,11 @@ const workspace = new Workspace(ROOT_PATH, [
path: path.join(ROOT_PATH, 'packages', 'playwright-chromium'),
files: LICENCE_FILES,
}),
new PWPackage({
name: '@playwright/experimental-tools',
path: path.join(ROOT_PATH, 'packages', 'playwright-tools'),
files: LICENCE_FILES,
}),
new PWPackage({
name: '@playwright/browser-webkit',
path: path.join(ROOT_PATH, 'packages', 'playwright-browser-webkit'),