From 657e435b6483a91d301c3b503cabb29948b9697a Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 16 Dec 2024 16:50:36 +0100 Subject: [PATCH] feat: download browsers as .tar.br --- .../bundles/zip/package-lock.json | 250 ++++++++++++++++++ .../playwright-core/bundles/zip/package.json | 2 + .../bundles/zip/src/zipBundleImpl.ts | 1 + .../server/registry/oopDownloadBrowserMain.ts | 5 +- packages/playwright-core/src/zipBundle.ts | 1 + 5 files changed, 256 insertions(+), 3 deletions(-) diff --git a/packages/playwright-core/bundles/zip/package-lock.json b/packages/playwright-core/bundles/zip/package-lock.json index 8476d11c19..9f78db5b21 100644 --- a/packages/playwright-core/bundles/zip/package-lock.json +++ b/packages/playwright-core/bundles/zip/package-lock.json @@ -9,10 +9,12 @@ "version": "0.0.1", "dependencies": { "extract-zip": "2.0.1", + "tar-fs": "^3.0.6", "yauzl": "2.10.0", "yazl": "2.5.1" }, "devDependencies": { + "@types/tar-fs": "^2.0.4", "@types/yauzl": "^2.10.0", "@types/yazl": "^2.4.2" } @@ -23,6 +25,27 @@ "integrity": "sha512-aveCYRQbgTH9Pssp1voEP7HiuWlD2jW2BO56w+bVrJn04i61yh6mRfoKO6hEYQD9vF+W8Chkwc6j1M36uPkx4g==", "devOptional": true }, + "node_modules/@types/tar-fs": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/tar-fs/-/tar-fs-2.0.4.tgz", + "integrity": "sha512-ipPec0CjTmVDWE+QKr9cTmIIoTl7dFG/yARCM5MqK8i6CNLIG1P8x4kwDsOQY1ChZOZjH0wO9nvfgBvWl4R3kA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/tar-stream": "*" + } + }, + "node_modules/@types/tar-stream": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-3.1.3.tgz", + "integrity": "sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz", @@ -41,6 +64,58 @@ "@types/node": "*" } }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, + "node_modules/bare-events": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.0.tgz", + "integrity": "sha512-/E8dDe9dsbLyh2qrZ64PEPadOQ0F4gbl1sUJOrmph7xOiIxfY8vwab/4bFLh4Y88/Hk/ujKcrQKc+ps0mv873A==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.5.tgz", + "integrity": "sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.0.0", + "bare-path": "^2.0.0", + "bare-stream": "^2.0.0" + } + }, + "node_modules/bare-os": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.4.tgz", + "integrity": "sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-path": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", + "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^2.1.0" + } + }, + "node_modules/bare-stream": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.1.tgz", + "integrity": "sha512-eVZbtKM+4uehzrsj49KtCy3Pbg7kO1pJ3SKZ1SFrIH/0pnj9scuGGgUlNDf/7qS8WKtGdiJY5Kyhs/ivYPTB/g==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + } + }, "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -92,6 +167,12 @@ "@types/yauzl": "^2.9.1" } }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", @@ -141,6 +222,60 @@ "once": "^1.3.1" } }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "license": "MIT" + }, + "node_modules/streamx": { + "version": "2.21.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.21.1.tgz", + "integrity": "sha512-PhP9wUnFLa+91CPy3N6tiQsK+gnYyUNuk15S3YG/zjYE7RuPeCjJngqnzpC31ow0lzBHQ+QGO4cNJnd0djYUsw==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, + "node_modules/tar-fs": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", + "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^2.1.1", + "bare-path": "^2.1.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -171,6 +306,25 @@ "integrity": "sha512-aveCYRQbgTH9Pssp1voEP7HiuWlD2jW2BO56w+bVrJn04i61yh6mRfoKO6hEYQD9vF+W8Chkwc6j1M36uPkx4g==", "devOptional": true }, + "@types/tar-fs": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/tar-fs/-/tar-fs-2.0.4.tgz", + "integrity": "sha512-ipPec0CjTmVDWE+QKr9cTmIIoTl7dFG/yARCM5MqK8i6CNLIG1P8x4kwDsOQY1ChZOZjH0wO9nvfgBvWl4R3kA==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/tar-stream": "*" + } + }, + "@types/tar-stream": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-3.1.3.tgz", + "integrity": "sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz", @@ -189,6 +343,52 @@ "@types/node": "*" } }, + "b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==" + }, + "bare-events": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.0.tgz", + "integrity": "sha512-/E8dDe9dsbLyh2qrZ64PEPadOQ0F4gbl1sUJOrmph7xOiIxfY8vwab/4bFLh4Y88/Hk/ujKcrQKc+ps0mv873A==", + "optional": true + }, + "bare-fs": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.5.tgz", + "integrity": "sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==", + "optional": true, + "requires": { + "bare-events": "^2.0.0", + "bare-path": "^2.0.0", + "bare-stream": "^2.0.0" + } + }, + "bare-os": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.4.tgz", + "integrity": "sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==", + "optional": true + }, + "bare-path": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", + "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", + "optional": true, + "requires": { + "bare-os": "^2.1.0" + } + }, + "bare-stream": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.1.tgz", + "integrity": "sha512-eVZbtKM+4uehzrsj49KtCy3Pbg7kO1pJ3SKZ1SFrIH/0pnj9scuGGgUlNDf/7qS8WKtGdiJY5Kyhs/ivYPTB/g==", + "optional": true, + "requires": { + "streamx": "^2.21.0" + } + }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -221,6 +421,11 @@ "yauzl": "^2.10.0" } }, + "fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, "fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", @@ -264,6 +469,51 @@ "once": "^1.3.1" } }, + "queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==" + }, + "streamx": { + "version": "2.21.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.21.1.tgz", + "integrity": "sha512-PhP9wUnFLa+91CPy3N6tiQsK+gnYyUNuk15S3YG/zjYE7RuPeCjJngqnzpC31ow0lzBHQ+QGO4cNJnd0djYUsw==", + "requires": { + "bare-events": "^2.2.0", + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" + } + }, + "tar-fs": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", + "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", + "requires": { + "bare-fs": "^2.1.1", + "bare-path": "^2.1.0", + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + } + }, + "tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "requires": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "requires": { + "b4a": "^1.6.4" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/packages/playwright-core/bundles/zip/package.json b/packages/playwright-core/bundles/zip/package.json index 0d9650d284..de339b69ca 100644 --- a/packages/playwright-core/bundles/zip/package.json +++ b/packages/playwright-core/bundles/zip/package.json @@ -10,10 +10,12 @@ }, "dependencies": { "extract-zip": "2.0.1", + "tar-fs": "^3.0.6", "yauzl": "2.10.0", "yazl": "2.5.1" }, "devDependencies": { + "@types/tar-fs": "^2.0.4", "@types/yauzl": "^2.10.0", "@types/yazl": "^2.4.2" } diff --git a/packages/playwright-core/bundles/zip/src/zipBundleImpl.ts b/packages/playwright-core/bundles/zip/src/zipBundleImpl.ts index b69e1a168a..ae0d67e79c 100644 --- a/packages/playwright-core/bundles/zip/src/zipBundleImpl.ts +++ b/packages/playwright-core/bundles/zip/src/zipBundleImpl.ts @@ -18,3 +18,4 @@ export * as yazl from 'yazl'; export * as yauzl from 'yauzl'; import extractZip from 'extract-zip'; export const extract = extractZip; +export * as tarFs from 'tar-fs'; diff --git a/packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts b/packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts index bd1b5f4fe7..d764fe2d9f 100644 --- a/packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts +++ b/packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts @@ -18,11 +18,10 @@ import fs from 'fs'; import path from 'path'; import { httpRequest } from '../../utils/network'; import { ManualPromise } from '../../utils/manualPromise'; -import { extract } from '../../zipBundle'; +import { extract, tarFs } from '../../zipBundle'; import type http from 'http'; import { pipeline } from 'stream/promises'; import { createBrotliDecompress } from 'zlib'; -import { TarExtractor } from '../../utils/tar'; export type DownloadParams = { title: string; @@ -151,7 +150,7 @@ async function downloadAndExtractBrotli(options: DownloadParams) { await pipeline( response, createBrotliDecompress(), - new TarExtractor(file => path.join(options.browserDirectory, file)), + tarFs.extract(options.browserDirectory) ); if (downloadedBytes !== totalBytes) diff --git a/packages/playwright-core/src/zipBundle.ts b/packages/playwright-core/src/zipBundle.ts index 9c275873a8..58d142533c 100644 --- a/packages/playwright-core/src/zipBundle.ts +++ b/packages/playwright-core/src/zipBundle.ts @@ -19,3 +19,4 @@ export type { ZipFile } from '../bundles/zip/node_modules/@types/yazl'; export const yauzl: typeof import('../bundles/zip/node_modules/@types/yauzl') = require('./zipBundleImpl').yauzl; export type { ZipFile as UnzipFile, Entry } from '../bundles/zip/node_modules/@types/yauzl'; export const extract: typeof import('../bundles/zip/node_modules/extract-zip') = require('./zipBundleImpl').extract; +export const tarFs: typeof import('../bundles/zip/node_modules/@types/tar-fs') = require('./zipBundleImpl').tarFs;