From bd2fdd0f2027435a131c1c5cc28517ae5b55c2de Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 27 Jan 2025 14:49:38 -0800 Subject: [PATCH] chore: land experimental tools (#34503) --- package-lock.json | 295 +++++++++++++++++- .../src/client/elementHandle.ts | 5 + .../playwright-core/src/client/locator.ts | 8 +- .../playwright-core/src/protocol/validator.ts | 6 + .../dispatchers/elementHandlerDispatcher.ts | 4 + packages/playwright-core/src/server/dom.ts | 15 +- packages/playwright-core/src/server/frames.ts | 4 +- .../src/server/injected/consoleApi.ts | 6 +- .../src/server/injected/injectedScript.ts | 14 +- .../playwright-core/src/server/selectors.ts | 2 +- packages/playwright-tools/.npmignore | 8 + packages/playwright-tools/README.md | 1 + packages/playwright-tools/browser.d.ts | 30 ++ packages/playwright-tools/browser.js | 19 ++ .../playwright-tools/computer-20241022.d.ts | 28 ++ .../playwright-tools/computer-20241022.js | 19 ++ packages/playwright-tools/package.json | 35 +++ .../src/examples/browser-anthropic.ts | 143 +++++++++ .../src/examples/browser-openai.ts | 161 ++++++++++ .../examples/computer-20241022-anthropic.ts | 148 +++++++++ .../playwright-tools/src/tools/browser.ts | 146 +++++++++ .../src/tools/computer-20241022.ts | 158 ++++++++++ packages/playwright-tools/src/tools/utils.ts | 71 +++++ packages/playwright-tools/types.d.ts | 27 ++ packages/protocol/src/channels.d.ts | 10 + packages/protocol/src/protocol.yml | 10 + utils/workspace.js | 5 + 27 files changed, 1362 insertions(+), 16 deletions(-) create mode 100644 packages/playwright-tools/.npmignore create mode 100644 packages/playwright-tools/README.md create mode 100644 packages/playwright-tools/browser.d.ts create mode 100644 packages/playwright-tools/browser.js create mode 100644 packages/playwright-tools/computer-20241022.d.ts create mode 100644 packages/playwright-tools/computer-20241022.js create mode 100644 packages/playwright-tools/package.json create mode 100644 packages/playwright-tools/src/examples/browser-anthropic.ts create mode 100644 packages/playwright-tools/src/examples/browser-openai.ts create mode 100644 packages/playwright-tools/src/examples/computer-20241022-anthropic.ts create mode 100644 packages/playwright-tools/src/tools/browser.ts create mode 100644 packages/playwright-tools/src/tools/computer-20241022.ts create mode 100644 packages/playwright-tools/src/tools/utils.ts create mode 100644 packages/playwright-tools/types.d.ts diff --git a/package-lock.json b/package-lock.json index d7eae56cba..c26ed0b1f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -111,6 +111,22 @@ "node": ">=6.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.33.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.33.1.tgz", + "integrity": "sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, "node_modules/@babel/cli": { "version": "7.23.4", "resolved": "https://registry.npmjs.org/@babel/cli/-/cli-7.23.4.tgz", @@ -1503,6 +1519,10 @@ "resolved": "packages/playwright-ct-vue", "link": true }, + "node_modules/@playwright/experimental-tools": { + "resolved": "packages/playwright-tools", + "link": true + }, "node_modules/@playwright/test": { "resolved": "packages/playwright-test", "link": true @@ -1867,6 +1887,17 @@ "undici-types": "~5.26.4" } }, + "node_modules/@types/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, "node_modules/@types/prop-types": { "version": "15.7.11", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.11.tgz", @@ -2331,6 +2362,19 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/acorn": { "version": "8.11.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", @@ -2351,6 +2395,19 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2590,6 +2647,13 @@ "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", "dev": true }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true, + "license": "MIT" + }, "node_modules/available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -2898,6 +2962,19 @@ "node": ">=0.1.90" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -3219,6 +3296,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -3930,6 +4017,16 @@ "node": ">=0.10.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/extract-zip": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", @@ -4077,6 +4174,42 @@ "is-callable": "^1.1.3" } }, + "node_modules/form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "dev": true, + "license": "MIT" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, "node_modules/formidable": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz", @@ -4562,6 +4695,16 @@ "node": ">=10.19.0" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/ignore": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", @@ -5358,6 +5501,29 @@ "node": ">=10.0.0" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-response": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", @@ -5438,6 +5604,47 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-releases": { "version": "2.0.14", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", @@ -5637,6 +5844,37 @@ "wrappy": "1" } }, + "node_modules/openai": { + "version": "4.79.1", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.79.1.tgz", + "integrity": "sha512-M7P5/PKnT/S/B5v0D64giC9mjyxFYkqlCuQFzR5hkdzMdqUuHf8T1gHhPGPF5oAvu4+PO3TvJv/qhZoS2bqAkw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/optionator": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", @@ -6754,6 +6992,13 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, "node_modules/trace-viewer": { "resolved": "packages/trace-viewer", "link": true @@ -7492,6 +7737,34 @@ "resolved": "packages/web", "link": true }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -7610,10 +7883,11 @@ "dev": true }, "node_modules/ws": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", - "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "dev": true, + "license": "MIT", "engines": { "node": ">=10.0.0" }, @@ -7932,6 +8206,21 @@ "node": ">=18" } }, + "packages/playwright-tools": { + "name": "@playwright/experimental-tools", + "version": "1.51.0-next", + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.51.0-next" + }, + "devDependencies": { + "@anthropic-ai/sdk": "^0.33.1", + "openai": "^4.79.1" + }, + "engines": { + "node": ">=18" + } + }, "packages/playwright-webkit": { "version": "1.51.0-next", "hasInstallScript": true, diff --git a/packages/playwright-core/src/client/elementHandle.ts b/packages/playwright-core/src/client/elementHandle.ts index 0a89d3091b..c6249e8078 100644 --- a/packages/playwright-core/src/client/elementHandle.ts +++ b/packages/playwright-core/src/client/elementHandle.ts @@ -62,6 +62,11 @@ export class ElementHandle extends JSHandle implements return Frame.fromNullable((await this._elementChannel.contentFrame()).frame); } + async _generateLocatorString(): Promise { + const value = (await this._elementChannel.generateLocatorString()).value; + return value === undefined ? null : value; + } + async getAttribute(name: string): Promise { const value = (await this._elementChannel.getAttribute({ name })).value; return value === undefined ? null : value; diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.ts index 84e68e1b20..7784e1d685 100644 --- a/packages/playwright-core/src/client/locator.ts +++ b/packages/playwright-core/src/client/locator.ts @@ -236,6 +236,10 @@ export class Locator implements api.Locator { return await this._frame._queryCount(this._selector); } + async _generateLocatorString(): Promise { + return await this._withElement(h => h._generateLocatorString()); + } + async getAttribute(name: string, options?: TimeoutOptions): Promise { return await this._frame.getAttribute(this._selector, name, { strict: true, ...options }); } @@ -288,8 +292,8 @@ export class Locator implements api.Locator { return await this._withElement((h, timeout) => h.screenshot({ ...options, timeout }), options.timeout); } - async ariaSnapshot(options?: TimeoutOptions): Promise { - const result = await this._frame._channel.ariaSnapshot({ ...options, selector: this._selector }); + async ariaSnapshot(options?: { _id?: boolean, _mode?: 'raw' | 'regex' } & TimeoutOptions): Promise { + const result = await this._frame._channel.ariaSnapshot({ ...options, id: options?._id, mode: options?._mode, selector: this._selector }); return result.snapshot; } diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 50e8b4f02a..cdcef14996 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -1429,6 +1429,8 @@ scheme.FrameAddStyleTagResult = tObject({ }); scheme.FrameAriaSnapshotParams = tObject({ selector: tString, + id: tOptional(tBoolean), + mode: tOptional(tEnum(['raw', 'regex'])), timeout: tOptional(tNumber), }); scheme.FrameAriaSnapshotResult = tObject({ @@ -1925,6 +1927,10 @@ scheme.ElementHandleFillParams = tObject({ scheme.ElementHandleFillResult = tOptional(tObject({})); scheme.ElementHandleFocusParams = tOptional(tObject({})); scheme.ElementHandleFocusResult = tOptional(tObject({})); +scheme.ElementHandleGenerateLocatorStringParams = tOptional(tObject({})); +scheme.ElementHandleGenerateLocatorStringResult = tObject({ + value: tOptional(tString), +}); scheme.ElementHandleGetAttributeParams = tObject({ name: tString, }); diff --git a/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts b/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts index d4d0280ec4..8a6e4fcbaa 100644 --- a/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/elementHandlerDispatcher.ts @@ -63,6 +63,10 @@ export class ElementHandleDispatcher extends JSHandleDispatcher implements chann return { frame: frame ? FrameDispatcher.from(this._browserContextDispatcher(), frame) : undefined }; } + async generateLocatorString(params: channels.ElementHandleGenerateLocatorStringParams, metadata: CallMetadata): Promise { + return { value: await this._elementHandle.generateLocatorString() }; + } + async getAttribute(params: channels.ElementHandleGetAttributeParams, metadata: CallMetadata): Promise { const value = await this._elementHandle.getAttribute(metadata, params.name); return { value: value === null ? undefined : value }; diff --git a/packages/playwright-core/src/server/dom.ts b/packages/playwright-core/src/server/dom.ts index 962f385c90..288b8c2c8d 100644 --- a/packages/playwright-core/src/server/dom.ts +++ b/packages/playwright-core/src/server/dom.ts @@ -28,7 +28,7 @@ import type { Progress } from './progress'; import { ProgressController } from './progress'; import type * as types from './types'; import type { TimeoutOptions } from '../common/types'; -import { isUnderTest } from '../utils'; +import { asLocator, isUnderTest } from '../utils'; import { prepareFilesForUpload } from './fileUploadUtils'; export type InputFilesItems = { @@ -185,6 +185,15 @@ export class ElementHandle extends js.JSHandle { return this._page._delegate.getContentFrame(this); } + async generateLocatorString(): Promise { + const selector = await this.evaluateInUtility(async ([injected, node]) => { + return injected.generateSelectorSimple(node as unknown as Element); + }, {}); + if (selector === 'error:notconnected') + return; + return asLocator('javascript', selector); + } + async getAttribute(metadata: CallMetadata, name: string): Promise { return this._frame.getAttribute(metadata, ':scope', name, {}, this); } @@ -799,8 +808,8 @@ export class ElementHandle extends js.JSHandle { return this._page._delegate.getBoundingBox(this); } - async ariaSnapshot(): Promise { - return await this.evaluateInUtility(([injected, element]) => injected.ariaSnapshot(element), {}); + async ariaSnapshot(options: { id?: boolean, mode?: 'raw' | 'regex' }): Promise { + return await this.evaluateInUtility(([injected, element, options]) => injected.ariaSnapshot(element, options), options); } async screenshot(metadata: CallMetadata, options: ScreenshotOptions & TimeoutOptions = {}): Promise { diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts index 1d2098b92c..1570df5770 100644 --- a/packages/playwright-core/src/server/frames.ts +++ b/packages/playwright-core/src/server/frames.ts @@ -1408,10 +1408,10 @@ export class Frame extends SdkObject { }); } - async ariaSnapshot(metadata: CallMetadata, selector: string, options: types.TimeoutOptions = {}): Promise { + async ariaSnapshot(metadata: CallMetadata, selector: string, options: { id?: boolean, mode?: 'raw' | 'regex' } & types.TimeoutOptions = {}): Promise { const controller = new ProgressController(metadata, this); return controller.run(async progress => { - return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot()); + return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => handle.ariaSnapshot(options)); }, this._page._timeoutSettings.timeout(options)); } diff --git a/packages/playwright-core/src/server/injected/consoleApi.ts b/packages/playwright-core/src/server/injected/consoleApi.ts index eae874834e..faba90c57d 100644 --- a/packages/playwright-core/src/server/injected/consoleApi.ts +++ b/packages/playwright-core/src/server/injected/consoleApi.ts @@ -85,10 +85,8 @@ class ConsoleAPI { inspect: (selector: string) => this._inspect(selector), selector: (element: Element) => this._selector(element), generateLocator: (element: Element, language?: Language) => this._generateLocator(element, language), - ariaSnapshot: (element?: Element) => { - const snapshot = this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body); - // eslint-disable-next-line no-console - console.log(snapshot); + ariaSnapshot: (element?: Element, options?: { id?: boolean }) => { + return this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body, options); }, resume: () => this._resume(), ...new Locator(injectedScript, ''), diff --git a/packages/playwright-core/src/server/injected/injectedScript.ts b/packages/playwright-core/src/server/injected/injectedScript.ts index 43988d4c28..bb004590f5 100644 --- a/packages/playwright-core/src/server/injected/injectedScript.ts +++ b/packages/playwright-core/src/server/injected/injectedScript.ts @@ -72,6 +72,7 @@ export class InjectedScript { // eslint-disable-next-line no-restricted-globals readonly window: Window & typeof globalThis; readonly document: Document; + private _ariaElementById: Map | undefined; // Recorder must use any external dependencies through InjectedScript. // Otherwise it will end up with a copy of all modules it uses, and any @@ -130,6 +131,7 @@ export class InjectedScript { this._engines.set('internal:attr', this._createNamedAttributeEngine()); this._engines.set('internal:testid', this._createNamedAttributeEngine()); this._engines.set('internal:role', createRoleEngine(true)); + this._engines.set('internal:aria-id', this._createAriaIdEngine()); for (const { name, engine } of customEngines) this._engines.set(name, engine); @@ -221,7 +223,8 @@ export class InjectedScript { if (node.nodeType !== Node.ELEMENT_NODE) throw this.createStacklessError('Can only capture aria snapshot of Element nodes.'); const ariaSnapshot = generateAriaTree(node as Element); - return renderAriaTree(ariaSnapshot.root, options); + this._ariaElementById = ariaSnapshot.elements; + return renderAriaTree(ariaSnapshot.root, { ...options, ids: options?.id ? ariaSnapshot.ids : undefined }); } ariaSnapshotAsObject(node: Node): AriaSnapshot { @@ -609,6 +612,15 @@ export class InjectedScript { return result; } + _createAriaIdEngine() { + const queryAll = (root: SelectorRoot, selector: string): Element[] => { + const ariaId = parseInt(selector, 10); + const result = this._ariaElementById?.get(ariaId); + return result && result.isConnected ? [result] : []; + }; + return { queryAll }; + } + elementState(node: Node, state: ElementStateWithoutStable): ElementStateQueryResult { const element = this.retarget(node, ['visible', 'hidden'].includes(state) ? 'none' : 'follow-label'); if (!element || !element.isConnected) { diff --git a/packages/playwright-core/src/server/selectors.ts b/packages/playwright-core/src/server/selectors.ts index b54a0269a5..edc11b9a86 100644 --- a/packages/playwright-core/src/server/selectors.ts +++ b/packages/playwright-core/src/server/selectors.ts @@ -39,7 +39,7 @@ export class Selectors { 'internal:has', 'internal:has-not', 'internal:has-text', 'internal:has-not-text', 'internal:and', 'internal:or', 'internal:chain', - 'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid', + 'role', 'internal:attr', 'internal:label', 'internal:text', 'internal:role', 'internal:testid', 'internal:aria-id' ]); this._builtinEnginesInMainWorld = new Set([ '_react', '_vue', diff --git a/packages/playwright-tools/.npmignore b/packages/playwright-tools/.npmignore new file mode 100644 index 0000000000..9b62c46c55 --- /dev/null +++ b/packages/playwright-tools/.npmignore @@ -0,0 +1,8 @@ +**/* +README.md +LICENSE +!lib/* +!browser.js +!browser.d.ts +!computer-20241022.js +!computer-20241022.d.ts diff --git a/packages/playwright-tools/README.md b/packages/playwright-tools/README.md new file mode 100644 index 0000000000..ce4a9dd7c6 --- /dev/null +++ b/packages/playwright-tools/README.md @@ -0,0 +1 @@ +> **BEWARE** This package is EXPERIMENTAL and does not respect semver. diff --git a/packages/playwright-tools/browser.d.ts b/packages/playwright-tools/browser.d.ts new file mode 100644 index 0000000000..e06589e947 --- /dev/null +++ b/packages/playwright-tools/browser.d.ts @@ -0,0 +1,30 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type playwright from 'playwright'; +import { ToolDeclaration, JSONSchemaType } from './types'; + +export type ToolResult = { + error?: string; + code: Array; + snapshot: string; +} + +export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise; + +export const schema: ToolDeclaration[]; +export const call: ToolCall; +export const snapshot: (page) => Promise; diff --git a/packages/playwright-tools/browser.js b/packages/playwright-tools/browser.js new file mode 100644 index 0000000000..d0475e384b --- /dev/null +++ b/packages/playwright-tools/browser.js @@ -0,0 +1,19 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { schema, call, snapshot } = require('./lib/tools/browser'); + +module.exports = { schema, call, snapshot }; diff --git a/packages/playwright-tools/computer-20241022.d.ts b/packages/playwright-tools/computer-20241022.d.ts new file mode 100644 index 0000000000..ce0f821d6d --- /dev/null +++ b/packages/playwright-tools/computer-20241022.d.ts @@ -0,0 +1,28 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type playwright from 'playwright'; +import { JSONSchemaType } from './types'; + +export type ToolResult = { + output?: string; + error?: string; + base64_image?: string; +}; + +export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise; + +export const call: ToolCall; diff --git a/packages/playwright-tools/computer-20241022.js b/packages/playwright-tools/computer-20241022.js new file mode 100644 index 0000000000..030d0e57e2 --- /dev/null +++ b/packages/playwright-tools/computer-20241022.js @@ -0,0 +1,19 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { call } = require('./lib/tools/computer-20241022'); + +module.exports = { call }; diff --git a/packages/playwright-tools/package.json b/packages/playwright-tools/package.json new file mode 100644 index 0000000000..c5366c2944 --- /dev/null +++ b/packages/playwright-tools/package.json @@ -0,0 +1,35 @@ +{ + "name": "@playwright/experimental-tools", + "version": "1.51.0-next", + "description": "Playwright Tools for AI", + "repository": { + "type": "git", + "url": "git+https://github.com/microsoft/playwright.git" + }, + "homepage": "https://playwright.dev", + "engines": { + "node": ">=18" + }, + "author": { + "name": "Microsoft Corporation" + }, + "license": "Apache-2.0", + "exports": { + "./browser": { + "types": "./browser.d.ts", + "default": "./browser.js" + }, + "./computer-20241022": { + "types": "./computer-20241022.d.ts", + "default": "./computer-20241022.js" + }, + "./package.json": "./package.json" + }, + "dependencies": { + "playwright": "1.51.0-next" + }, + "devDependencies": { + "@anthropic-ai/sdk": "^0.33.1", + "openai": "^4.79.1" + } +} diff --git a/packages/playwright-tools/src/examples/browser-anthropic.ts b/packages/playwright-tools/src/examples/browser-anthropic.ts new file mode 100644 index 0000000000..0053c5749d --- /dev/null +++ b/packages/playwright-tools/src/examples/browser-anthropic.ts @@ -0,0 +1,143 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable no-console */ + +import playwright from 'playwright'; +import Anthropic from '@anthropic-ai/sdk'; +import dotenv from 'dotenv'; +import browser from '@playwright/experimental-tools/browser'; + +dotenv.config(); + +const anthropic = new Anthropic(); + +export const system = ` +You are a web tester. + + +- Perform test according to the provided checklist +- Use browser tools to perform actions on web page +- Never ask questions, always perform a best guess action +- Use one tool at a time, wait for its result before proceeding. +- When ready use "reportResult" tool to report result +`; + +const reportTool: Anthropic.Tool = { + name: 'reportResult', + description: 'Submit test result', + input_schema: { + type: 'object', + properties: { + 'success': { type: 'boolean', description: 'Whether test passed' }, + 'result': { type: 'string', description: 'Result of the test if some information has been requested' }, + 'error': { type: 'string', description: 'Error message if test failed' } + }, + required: ['success'] + } +}; + +type Message = Anthropic.Beta.Messages.BetaMessageParam & { + history: Anthropic.Beta.Messages.BetaMessageParam['content'] +}; + +async function anthropicAgentLoop(page: playwright.Page, task: string) { + // Convert them into tools for Anthropic. + const pageTools: Anthropic.Tool[] = browser.schema.map(tool => { + return { + name: tool.name, + description: tool.description, + input_schema: tool.parameters as any, + }; + }); + + // Add report tool. + const tools = [reportTool, ...pageTools]; + + const history: Message[] = [{ + role: 'user', + history: `Task: ${task}`, + content: `Task: ${task}\n\n${await browser.snapshot(page)}`, + }]; + + // Run agentic loop, cap steps. + for (let i = 0; i < 50; i++) { + const response = await anthropic.messages.create({ + model: 'claude-3-5-sonnet-20241022', + max_tokens: 1024, + temperature: 0, + tools, + system, + messages: toAnthropicMessages(history), + }); + history.push({ role: 'assistant', content: response.content, history: response.content }); + + const toolUse = response.content.find(block => block.type === 'tool_use'); + if (!toolUse) { + history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' }); + continue; + } + + if (toolUse.name === 'reportResult') { + console.log(toolUse.input); + return; + } + + // Run the Playwright tool. + const { error, snapshot, code } = await browser.call(page, toolUse.name, toolUse.input as any); + if (code.length) + console.log(code.join('\n')); + + // Report the result. + const resultText = error ? `Error: ${error}\n` : 'Done\n'; + history.push({ + role: 'user', + content: [{ + type: 'tool_result', + tool_use_id: toolUse.id, + content: [{ type: 'text', text: resultText + snapshot }], + }], + history: [{ + type: 'tool_result', + tool_use_id: toolUse.id, + content: [{ type: 'text', text: resultText }], + }], + }); + } +} + +function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] { + return messages.map((message, i) => { + if (i === messages.length - 1) + return { ...message, history: undefined }; + return { ...message, content: message.history, history: undefined }; + }); +} + +async function main() { + const browser = await playwright.chromium.launch({ headless: false }); + const page = await browser.newPage(); + await anthropicAgentLoop(page, ` + - Go to http://github.com/microsoft + - Search for "playwright" repository + - Navigate to it + - Switch into the Issues tab + - Report 3 first issues + `); + await browser.close(); +} + +void main(); diff --git a/packages/playwright-tools/src/examples/browser-openai.ts b/packages/playwright-tools/src/examples/browser-openai.ts new file mode 100644 index 0000000000..3d39618f5b --- /dev/null +++ b/packages/playwright-tools/src/examples/browser-openai.ts @@ -0,0 +1,161 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable no-console */ + +import playwright from 'playwright'; +import browser from '@playwright/experimental-tools/browser'; +import dotenv from 'dotenv'; +import OpenAI from 'openai'; +import type { ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources'; + +dotenv.config(); + +const openai = new OpenAI(); + +export const system = ` +You are a web tester. + +to +- Perform test according to the provided checklist +- Use browser tools to perform actions on web page +- Never ask questions, always perform a best guess action +- When ready use "reportResult" tool to report result +- You can only make one tool call at a time. +`; + +type Message = ChatCompletionMessageParam & { + history: any +}; + +const reportTool: ChatCompletionTool = { + type: 'function', + function: { + name: 'reportResult', + description: 'Submit test result', + parameters: { + type: 'object', + properties: { + success: { type: 'boolean', description: 'Whether test passed' }, + result: { type: 'string', description: 'Result of the test if requested' }, + error: { type: 'string', description: 'Error if test failed' }, + }, + required: ['success'], + additionalProperties: false, + }, + } +}; + +async function openAIAgentLoop(page: playwright.Page, task: string) { + const pageTools: ChatCompletionTool[] = browser.schema.map(tool => ({ + type: 'function', + function: { + name: tool.name, + description: tool.description, + parameters: { + ...tool.parameters, + additionalProperties: false, + }, + } + })); + + const tools = [reportTool, ...pageTools]; + + const history: Message[] = [ + { + role: 'system', content: system, history: system + }, + { + role: 'user', + history: `Task: ${task}`, + content: `Task: ${task}\n\n${await browser.snapshot(page)}`, + } + ]; + + // Run agentic loop, cap steps. + for (let i = 0; i < 50; i++) { + const completion = await openai.chat.completions.create({ + model: 'gpt-4o', + messages: toOpenAIMessages(history), + tools, + store: true, + }); + + console.log(JSON.stringify(completion, null, 2)); + + const toolCalls = completion.choices[0]?.message?.tool_calls; + if (!toolCalls || toolCalls.length !== 1 || toolCalls[0].type !== 'function') { + history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' }); + continue; + } + + const toolCall = toolCalls[0]; + if (toolCall.function.name === 'reportResult') { + console.log(JSON.parse(toolCall.function.arguments)); + return; + } + + history.push({ ...completion.choices[0].message, history: null }); + + // Run the Playwright tool. + const params = JSON.parse(toolCall.function.arguments); + const { error, snapshot, code } = await browser.call(page, toolCall.function.name, params); + console.log({ error, code, snapshot }); + if (code.length) + console.log(code.join('\n')); + + if (toolCall.function.name === 'log') + return; + + // Report the result. + const resultText = error ? `Error: ${error}\n` : 'Done\n'; + history.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: resultText + snapshot, + history: resultText, + }); + } +} + +function toOpenAIMessages(messages: Message[]): ChatCompletionMessageParam[] { + return messages.map((message, i) => { + const copy: Message = { ...message }; + delete copy.history; + if (i === messages.length - 1) + return copy; + copy.content = message.history; + return copy; + }); +} + +async function main() { + const browser = await playwright.chromium.launch({ headless: false }); + const page = await browser.newPage(); + await openAIAgentLoop(page, ` + - Go to http://github.com/microsoft + - Search for "playwright" repository + - Navigate to it + - Capture snapshot for toolbar with Code, Issues, etc. + - Capture snapshot for branch selector + - Assert that number of Issues is present + - Switch into the Issues tab + - Report 3 first issues + `); + await browser.close(); +} + +void main(); diff --git a/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts b/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts new file mode 100644 index 0000000000..20028c564c --- /dev/null +++ b/packages/playwright-tools/src/examples/computer-20241022-anthropic.ts @@ -0,0 +1,148 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable no-console */ + +import playwright from 'playwright'; +import Anthropic from '@anthropic-ai/sdk'; +import dotenv from 'dotenv'; +import computer, { type ToolResult } from '@playwright/experimental-tools/computer-20241022'; +import type { BetaImageBlockParam, BetaTextBlockParam } from '@anthropic-ai/sdk/resources/beta/messages/messages'; + +dotenv.config(); + +const anthropic = new Anthropic(); + +export const system = ` +You are a web tester. + + +- Perform test according to the provided checklist +- Use browser tools to perform actions on web page +- Never ask questions, always perform a best guess action +- Use one tool at a time, wait for its result before proceeding. +- When ready use "reportResult" tool to report result +`; + +const computerTool: Anthropic.Beta.BetaToolUnion = { + type: 'computer_20241022', + name: 'computer', + display_width_px: 1920, + display_height_px: 1080, + display_number: 1, +}; + +const reportTool: Anthropic.Tool = { + name: 'reportResult', + description: 'Submit test result', + input_schema: { + type: 'object', + properties: { + 'success': { type: 'boolean', description: 'Whether test passed' }, + 'result': { type: 'string', description: 'Result of the test if some information has been requested' }, + 'error': { type: 'string', description: 'Error message if test failed' } + }, + required: ['success'] + } +}; + +type Message = Anthropic.Beta.Messages.BetaMessageParam & { + history: Anthropic.Beta.Messages.BetaMessageParam['content'] +}; + +async function anthropicAgentLoop(page: playwright.Page, task: string) { + // Add report tool. + const tools = [reportTool, computerTool]; + + const history: Message[] = [{ + role: 'user', + history: `Task: ${task}`, + content: `Task: ${task}`, + }]; + + // Run agentic loop, cap steps. + for (let i = 0; i < 50; i++) { + const response = await anthropic.beta.messages.create({ + model: 'claude-3-5-sonnet-20241022', + max_tokens: 1024, + temperature: 0, + tools, + system, + messages: toAnthropicMessages(history), + betas: ['computer-use-2024-10-22'], + }); + + history.push({ role: 'assistant', content: response.content, history: response.content }); + + const toolUse = response.content.find(block => block.type === 'tool_use'); + if (!toolUse) { + history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' }); + continue; + } + + if (toolUse.name === 'reportResult') { + console.log(toolUse.input); + return; + } + + const result: ToolResult = await computer.call(page, toolUse.name, toolUse.input as any); + const contentEntry: BetaTextBlockParam | BetaImageBlockParam = result.base64_image ? { + type: 'image', + source: { type: 'base64', media_type: 'image/jpeg', data: result.base64_image } + } : { + type: 'text', + text: result.output || '', + }; + history.push({ + role: 'user', + content: [{ + type: 'tool_result', + tool_use_id: toolUse.id, + content: [contentEntry], + }], + history: [{ + type: 'tool_result', + tool_use_id: toolUse.id, + content: [{ type: 'text', text: '' }], + }], + }); + } +} + +function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] { + return messages.map((message, i) => { + if (i === messages.length - 1) + return { ...message, history: undefined }; + return { ...message, content: message.history, history: undefined }; + }); +} + +const githubTask = ` + - Search for "playwright" repository + - Navigate to it + - Switch into the Issues tab + - Report 3 first issues +`; + +async function main() { + const browser = await playwright.chromium.launch({ headless: false }); + const page = await browser.newPage(); + await page.goto('http://github.com/microsoft'); + await anthropicAgentLoop(page, githubTask); + await browser.close(); +} + +void main(); diff --git a/packages/playwright-tools/src/tools/browser.ts b/packages/playwright-tools/src/tools/browser.ts new file mode 100644 index 0000000000..267d03bcb3 --- /dev/null +++ b/packages/playwright-tools/src/tools/browser.ts @@ -0,0 +1,146 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type playwright from 'playwright'; +import type { JSONSchemaType, ToolDeclaration } from '../../types'; +import type { ToolResult } from '../../browser'; +import { waitForNetwork } from './utils'; + +type LocatorEx = playwright.Locator & { + _generateLocatorString: () => Promise; +}; + +const intentProperty = { + intent: { + type: 'string', + description: 'Intent behind this particular action. Used as a comment.', + } +}; + +const elementIdProperty = { + elementId: { + type: 'number', + description: 'Target element', + } +}; + +export const schema: ToolDeclaration[] = [ + { + name: 'navigate', + description: 'Navigate to a URL', + parameters: { + type: 'object', + properties: { + ...intentProperty, + url: { + type: 'string', + description: 'URL to navigate to', + }, + }, + required: ['intent', 'elementId'], + } + }, + { + name: 'click', + description: 'Perform click on a web page', + parameters: { + type: 'object', + properties: { + ...intentProperty, + ...elementIdProperty, + }, + required: ['intent', 'elementId'], + } + }, + { + name: 'enterText', + description: 'Enter text into editable element', + parameters: { + type: 'object', + properties: { + ...intentProperty, + ...elementIdProperty, + text: { + type: 'string', + description: 'Text to enter', + }, + submit: { + type: 'boolean', + description: 'Whether to submit entered text (press Enter after)' + } + }, + required: ['intent', 'elementId', 'text'], + } + }, + { + name: 'wait', + description: `Wait for given amount of time to see if the page updates. Use it after action if you think page is not ready yet`, + parameters: { + type: 'object', + properties: { + ...intentProperty, + time: { + type: 'integer', + description: 'Time to wait in seconds', + }, + }, + required: ['intent', 'time'], + } + }, +]; + +export async function call(page: playwright.Page, toolName: string, params: Record): Promise { + const code: string[] = []; + try { + await waitForNetwork(page, async () => { + await performAction(page, toolName, params, code); + }); + } catch (e) { + return { error: e.message, snapshot: await snapshot(page), code }; + } + return { snapshot: await snapshot(page), code }; +} + +export async function snapshot(page: playwright.Page) { + const params = { _id: true } as any; + return `\n${await page.locator('body').ariaSnapshot(params)}\n`; +} + +async function performAction(page: playwright.Page, toolName: string, params: Record, code: string[]) { + const locator = elementLocator(page, params); + code.push((params.intent as string).split('\n').map(line => `// ${line}`).join('\n')); + if (toolName === 'navigate') { + code.push(`await page.goto(${JSON.stringify(params.url)})`); + await page.goto(params.url as string); + } else if (toolName === 'click') { + code.push(`await page.${await locator._generateLocatorString()}.click()`); + await locator.click(); + } else if (toolName === 'enterText') { + code.push(`await page.${await locator._generateLocatorString()}.click()`); + await locator.click(); + code.push(`await page.${await locator._generateLocatorString()}.fill(${JSON.stringify(params.text)})`); + await locator.fill(params.text as string); + if (params.submit) { + code.push(`await page.${await locator._generateLocatorString()}.press("Enter")`); + await locator.press('Enter'); + } + } +} + +function elementLocator(page: playwright.Page, params: any): LocatorEx { + return page.locator(`internal:aria-id=${params.elementId}`) as LocatorEx; +} diff --git a/packages/playwright-tools/src/tools/computer-20241022.ts b/packages/playwright-tools/src/tools/computer-20241022.ts new file mode 100644 index 0000000000..e407d027ba --- /dev/null +++ b/packages/playwright-tools/src/tools/computer-20241022.ts @@ -0,0 +1,158 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type playwright from 'playwright'; +import type { JSONSchemaType } from '../../types'; +import type { ToolResult } from '../../computer-20241022'; +import { waitForNetwork } from './utils'; + +export async function call(page: playwright.Page, toolName: string, input: Record): Promise { + if (toolName !== 'computer') + throw new Error('Unsupported tool'); + return await waitForNetwork(page, async () => { + return await performAction(page, toolName, input); + }); +} + +type PageState = { + x: number; + y: number; +}; + +const pageStateSymbol = Symbol('pageState'); + +function pageState(page: playwright.Page): PageState { + if (!(page as any)[pageStateSymbol]) + (page as any)[pageStateSymbol] = { x: 0, y: 0 }; + return (page as any)[pageStateSymbol]; +} + +async function performAction(page: playwright.Page, toolName: string, input: Record): Promise { + const state = pageState(page); + const { action } = input as { action: string }; + if (action === 'screenshot') { + const screenshot = await page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' }); + return { + output: 'Screenshot', + base64_image: screenshot.toString('base64'), + }; + } + if (action === 'mouse_move') { + const { coordinate } = input as { coordinate: [number, number] }; + state.x = coordinate[0]; + state.y = coordinate[1]; + await page.mouse.move(state.x, state.y); + return { output: 'Mouse moved' }; + } + if (action === 'left_click') { + await page.mouse.down(); + await page.mouse.up(); + return { output: 'Left clicked' }; + } + if (action === 'left_click_drag') { + await page.mouse.down(); + const { coordinate } = input as { coordinate: [number, number] }; + state.x = coordinate[0]; + state.y = coordinate[1]; + await page.mouse.move(state.x, state.y); + await page.mouse.up(); + return { output: 'Left dragged' }; + } + if (action === 'right_click') { + await page.mouse.down({ button: 'right' }); + await page.mouse.up({ button: 'right' }); + return { output: 'Right clicked' }; + } + if (action === 'double_click') { + await page.mouse.down(); + await page.mouse.up(); + await page.mouse.down(); + await page.mouse.up(); + return { output: 'Double clicked' }; + } + if (action === 'middle_click') { + await page.mouse.down({ button: 'middle' }); + await page.mouse.up({ button: 'middle' }); + return { output: 'Middle clicked' }; + } + if (action === 'key') { + const { text } = input as { text: string }; + await page.keyboard.press(xToPlaywright(text)); + return { output: 'Text typed' }; + } + if (action === 'cursor_position') + return { output: `X=${state.x},Y=${state.y}` }; + throw new Error('Unimplemented tool: ' + toolName); +} + +const xToPlaywrightKeyMap = new Map([ + ['BackSpace', 'Backspace'], + ['Tab', 'Tab'], + ['Return', 'Enter'], + ['Escape', 'Escape'], + ['space', ' '], + ['Delete', 'Delete'], + ['Home', 'Home'], + ['End', 'End'], + ['Left', 'ArrowLeft'], + ['Up', 'ArrowUp'], + ['Right', 'ArrowRight'], + ['Down', 'ArrowDown'], + ['Insert', 'Insert'], + ['Page_Up', 'PageUp'], + ['Page_Down', 'PageDown'], + ['F1', 'F1'], + ['F2', 'F2'], + ['F3', 'F3'], + ['F4', 'F4'], + ['F5', 'F5'], + ['F6', 'F6'], + ['F7', 'F7'], + ['F8', 'F8'], + ['F9', 'F9'], + ['F10', 'F10'], + ['F11', 'F11'], + ['F12', 'F12'], + ['Shift_L', 'Shift'], + ['Shift_R', 'Shift'], + ['Control_L', 'Control'], + ['Control_R', 'Control'], + ['Alt_L', 'Alt'], + ['Alt_R', 'Alt'], + ['Super_L', 'Meta'], + ['Super_R', 'Meta'], +]); + +const xToPlaywrightModifierMap = new Map([ + ['alt', 'Alt'], + ['control', 'Control'], + ['meta', 'Meta'], + ['shift', 'Shift'], +]); + + +const xToPlaywright = (key: string) => { + const tokens = key.split('+'); + if (tokens.length === 1) + return xToPlaywrightKeyMap.get(key) || key; + if (tokens.length === 2) { + const modifier = xToPlaywrightModifierMap.get(tokens[0]); + const key = xToPlaywrightKeyMap.get(tokens[1]) || tokens[1]; + return modifier + '+' + key; + } + throw new Error('Invalid key: ' + key); +}; diff --git a/packages/playwright-tools/src/tools/utils.ts b/packages/playwright-tools/src/tools/utils.ts new file mode 100644 index 0000000000..08e674014d --- /dev/null +++ b/packages/playwright-tools/src/tools/utils.ts @@ -0,0 +1,71 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * Modifications copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type playwright from 'playwright'; +import { ManualPromise } from 'playwright-core/lib/utils'; + +export async function waitForNetwork(page: playwright.Page, callback: () => Promise): Promise { + const requests = new Set(); + let frameNavigated = false; + const waitBarrier = new ManualPromise(); + + const requestListener = (request: playwright.Request) => requests.add(request); + const requestFinishedListener = (request: playwright.Request) => { + requests.delete(request); + if (!requests.size) + waitBarrier.resolve(); + }; + + const frameNavigateListener = (frame: playwright.Frame) => { + if (frame.parentFrame()) + return; + frameNavigated = true; + dispose(); + clearTimeout(timeout); + void frame.waitForLoadState('load').then(() => { + waitBarrier.resolve(); + }); + }; + + const onTimeout = () => { + dispose(); + waitBarrier.resolve(); + }; + + page.on('request', requestListener); + page.on('requestfinished', requestFinishedListener); + page.on('framenavigated', frameNavigateListener); + const timeout = setTimeout(onTimeout, 10000); + + const dispose = () => { + page.off('request', requestListener); + page.off('requestfinished', requestFinishedListener); + page.off('framenavigated', frameNavigateListener); + clearTimeout(timeout); + }; + + try { + const result = await callback(); + if (!requests.size && !frameNavigated) + waitBarrier.resolve(); + await waitBarrier; + await page.evaluate(() => new Promise(f => setTimeout(f, 1000))); + return result; + } finally { + dispose(); + } +} diff --git a/packages/playwright-tools/types.d.ts b/packages/playwright-tools/types.d.ts new file mode 100644 index 0000000000..5fe61526c2 --- /dev/null +++ b/packages/playwright-tools/types.d.ts @@ -0,0 +1,27 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type playwright from 'playwright'; + +export type JSONSchemaType = string | number | boolean | JSONSchemaObject | JSONSchemaArray | null; +interface JSONSchemaObject { [key: string]: JSONSchemaType; } +interface JSONSchemaArray extends Array {} + +export type ToolDeclaration = { + name: string; + description: string; + parameters: any; +}; diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index 526cc599ab..30f8c03088 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -2619,9 +2619,13 @@ export type FrameAddStyleTagResult = { }; export type FrameAriaSnapshotParams = { selector: string, + id?: boolean, + mode?: 'raw' | 'regex', timeout?: number, }; export type FrameAriaSnapshotOptions = { + id?: boolean, + mode?: 'raw' | 'regex', timeout?: number, }; export type FrameAriaSnapshotResult = { @@ -3311,6 +3315,7 @@ export interface ElementHandleChannel extends ElementHandleEventTarget, JSHandle dispatchEvent(params: ElementHandleDispatchEventParams, metadata?: CallMetadata): Promise; fill(params: ElementHandleFillParams, metadata?: CallMetadata): Promise; focus(params?: ElementHandleFocusParams, metadata?: CallMetadata): Promise; + generateLocatorString(params?: ElementHandleGenerateLocatorStringParams, metadata?: CallMetadata): Promise; getAttribute(params: ElementHandleGetAttributeParams, metadata?: CallMetadata): Promise; hover(params: ElementHandleHoverParams, metadata?: CallMetadata): Promise; innerHTML(params?: ElementHandleInnerHTMLParams, metadata?: CallMetadata): Promise; @@ -3450,6 +3455,11 @@ export type ElementHandleFillResult = void; export type ElementHandleFocusParams = {}; export type ElementHandleFocusOptions = {}; export type ElementHandleFocusResult = void; +export type ElementHandleGenerateLocatorStringParams = {}; +export type ElementHandleGenerateLocatorStringOptions = {}; +export type ElementHandleGenerateLocatorStringResult = { + value?: string, +}; export type ElementHandleGetAttributeParams = { name: string, }; diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index df54dcbe1c..77501efa9b 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -1879,6 +1879,12 @@ Frame: ariaSnapshot: parameters: selector: string + id: boolean? + mode: + type: enum? + literals: + - raw + - regex timeout: number? returns: snapshot: string @@ -2648,6 +2654,10 @@ ElementHandle: slowMo: true snapshot: true + generateLocatorString: + returns: + value: string? + getAttribute: parameters: name: string diff --git a/utils/workspace.js b/utils/workspace.js index 4fb69050b3..7e6850a262 100755 --- a/utils/workspace.js +++ b/utils/workspace.js @@ -167,6 +167,11 @@ const workspace = new Workspace(ROOT_PATH, [ path: path.join(ROOT_PATH, 'packages', 'playwright-chromium'), files: LICENCE_FILES, }), + new PWPackage({ + name: '@playwright/experimental-tools', + path: path.join(ROOT_PATH, 'packages', 'playwright-tools'), + files: LICENCE_FILES, + }), new PWPackage({ name: '@playwright/browser-webkit', path: path.join(ROOT_PATH, 'packages', 'playwright-browser-webkit'),