From 1f05f90bcd2e02babf854b08cc823ce5de01f57f Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 29 Nov 2024 11:55:31 +0100 Subject: [PATCH] allow parallel downloads --- .../bundles/utils/package-lock.json | 44 +++++++++++++++++-- .../bundles/utils/package.json | 4 +- .../bundles/utils/src/utilsBundleImpl.ts | 4 +- .../src/server/registry/browserFetcher.ts | 22 +++++----- packages/playwright-core/src/utilsBundle.ts | 2 +- 5 files changed, 58 insertions(+), 18 deletions(-) diff --git a/packages/playwright-core/bundles/utils/package-lock.json b/packages/playwright-core/bundles/utils/package-lock.json index 6d9b0562e3..d15b87b960 100644 --- a/packages/playwright-core/bundles/utils/package-lock.json +++ b/packages/playwright-core/bundles/utils/package-lock.json @@ -18,9 +18,9 @@ "jpeg-js": "0.4.4", "mime": "^3.0.0", "minimatch": "^3.1.2", + "multi-progress": "^4.0.0", "open": "8.4.0", "pngjs": "6.0.0", - "progress": "2.0.3", "proxy-from-env": "1.1.0", "retry": "0.12.0", "signal-exit": "3.0.7", @@ -34,8 +34,8 @@ "@types/diff": "^6.0.0", "@types/mime": "^2.0.3", "@types/minimatch": "^3.0.5", + "@types/multi-progress": "^2.0.6", "@types/pngjs": "^6.0.1", - "@types/progress": "^2.0.5", "@types/proper-lockfile": "^4.1.2", "@types/proxy-from-env": "^1.0.1", "@types/stack-utils": "^2.0.1", @@ -76,6 +76,17 @@ "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==", "dev": true }, + "node_modules/@types/multi-progress": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/multi-progress/-/multi-progress-2.0.6.tgz", + "integrity": "sha512-S+DW6NQdvwJ6liFUmGNbGtUwFwHm/qx2D2hrlX5HY5f55hRpteS1F1pHfrC2AoCvPNAeVuFVlZTDyIkPssMujg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/progress": "*" + } + }, "node_modules/@types/node": { "version": "17.0.25", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.25.tgz", @@ -330,6 +341,15 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/multi-progress": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/multi-progress/-/multi-progress-4.0.0.tgz", + "integrity": "sha512-9zcjyOou3FFCKPXsmkbC3ethv51SFPoA4dJD6TscIp2pUmy26kBDZW6h9XofPELrzseSkuD7r0V+emGEeo39Pg==", + "license": "MIT", + "peerDependencies": { + "progress": "^2.0.0" + } + }, "node_modules/open": { "version": "8.4.0", "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz", @@ -358,6 +378,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "peer": true, "engines": { "node": ">=0.4.0" } @@ -498,6 +519,16 @@ "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==", "dev": true }, + "@types/multi-progress": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/multi-progress/-/multi-progress-2.0.6.tgz", + "integrity": "sha512-S+DW6NQdvwJ6liFUmGNbGtUwFwHm/qx2D2hrlX5HY5f55hRpteS1F1pHfrC2AoCvPNAeVuFVlZTDyIkPssMujg==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/progress": "*" + } + }, "@types/node": { "version": "17.0.25", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.25.tgz", @@ -690,6 +721,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "multi-progress": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/multi-progress/-/multi-progress-4.0.0.tgz", + "integrity": "sha512-9zcjyOou3FFCKPXsmkbC3ethv51SFPoA4dJD6TscIp2pUmy26kBDZW6h9XofPELrzseSkuD7r0V+emGEeo39Pg==", + "requires": {} + }, "open": { "version": "8.4.0", "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz", @@ -708,7 +745,8 @@ "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "peer": true }, "proxy-from-env": { "version": "1.1.0", diff --git a/packages/playwright-core/bundles/utils/package.json b/packages/playwright-core/bundles/utils/package.json index 1900d423f5..ee362eecdd 100644 --- a/packages/playwright-core/bundles/utils/package.json +++ b/packages/playwright-core/bundles/utils/package.json @@ -19,9 +19,9 @@ "jpeg-js": "0.4.4", "mime": "^3.0.0", "minimatch": "^3.1.2", + "multi-progress": "^4.0.0", "open": "8.4.0", "pngjs": "6.0.0", - "progress": "2.0.3", "proxy-from-env": "1.1.0", "retry": "0.12.0", "signal-exit": "3.0.7", @@ -35,8 +35,8 @@ "@types/diff": "^6.0.0", "@types/mime": "^2.0.3", "@types/minimatch": "^3.0.5", + "@types/multi-progress": "^2.0.6", "@types/pngjs": "^6.0.1", - "@types/progress": "^2.0.5", "@types/proper-lockfile": "^4.1.2", "@types/proxy-from-env": "^1.0.1", "@types/stack-utils": "^2.0.1", diff --git a/packages/playwright-core/bundles/utils/src/utilsBundleImpl.ts b/packages/playwright-core/bundles/utils/src/utilsBundleImpl.ts index 5c8434f907..63da4d5a35 100644 --- a/packages/playwright-core/bundles/utils/src/utilsBundleImpl.ts +++ b/packages/playwright-core/bundles/utils/src/utilsBundleImpl.ts @@ -49,8 +49,8 @@ export { PNG } from 'pngjs'; export { program } from 'commander'; -import progressLibrary from 'progress'; -export const progress = progressLibrary; +import multiProgressLibrary from 'multi-progress'; +export const multiProgress = multiProgressLibrary; export { SocksProxyAgent } from 'socks-proxy-agent'; diff --git a/packages/playwright-core/src/server/registry/browserFetcher.ts b/packages/playwright-core/src/server/registry/browserFetcher.ts index 3a6e36b42d..0b8ce0210d 100644 --- a/packages/playwright-core/src/server/registry/browserFetcher.ts +++ b/packages/playwright-core/src/server/registry/browserFetcher.ts @@ -22,7 +22,7 @@ import childProcess from 'child_process'; import { existsAsync } from '../../utils/fileUtils'; import { debugLogger } from '../../utils/debugLogger'; import { ManualPromise } from '../../utils/manualPromise'; -import { colors, progress as ProgressBar } from '../../utilsBundle'; +import { colors, multiProgress as MultiProgressBar } from '../../utilsBundle'; import { browserDirectoryToMarkerFilePath } from '.'; import { getUserAgent } from '../../utils/userAgent'; import type { DownloadParams } from './oopDownloadBrowserMain'; @@ -75,7 +75,7 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirectory: string, url: string, zipPath: string, executablePath: string | undefined, connectionTimeout: number): Promise<{ error: Error | null }> { const cp = childProcess.fork(path.join(__dirname, 'oopDownloadBrowserMain.js')); const promise = new ManualPromise<{ error: Error | null }>(); - const progress = getDownloadProgress(); + const progress = getDownloadProgress(title); cp.on('message', (message: any) => { if (message?.method === 'log') debugLogger.log('install', message.params.message); @@ -122,20 +122,22 @@ export function logPolitely(toBeLogged: string) { type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; -function getDownloadProgress(): OnProgressCallback { +function getDownloadProgress(title: string): OnProgressCallback { if (process.stdout.isTTY) - return getAnimatedDownloadProgress(); - return getBasicDownloadProgress(); + return getAnimatedDownloadProgress(title); + return getBasicDownloadProgress(title); } -function getAnimatedDownloadProgress(): OnProgressCallback { +const multiProgress = new MultiProgressBar(); + +function getAnimatedDownloadProgress(title: string): OnProgressCallback { let progressBar: ProgressBar; let lastDownloadedBytes = 0; return (downloadedBytes: number, totalBytes: number) => { if (!progressBar) { - progressBar = new ProgressBar( - `${toMegabytes( + progressBar = multiProgress.newBar( + `${title} ${toMegabytes( totalBytes )} [:bar] :percent :etas`, { @@ -152,7 +154,7 @@ function getAnimatedDownloadProgress(): OnProgressCallback { }; } -function getBasicDownloadProgress(): OnProgressCallback { +function getBasicDownloadProgress(title: string): OnProgressCallback { const totalRows = 10; const stepWidth = 8; let lastRow = -1; @@ -163,7 +165,7 @@ function getBasicDownloadProgress(): OnProgressCallback { lastRow = row; const percentageString = String(percentage * 100 | 0).padStart(3); // eslint-disable-next-line no-console - console.log(`|${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`); + console.log(`${title} |${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`); } }; } diff --git a/packages/playwright-core/src/utilsBundle.ts b/packages/playwright-core/src/utilsBundle.ts index 72bcee397e..f4d9215209 100644 --- a/packages/playwright-core/src/utilsBundle.ts +++ b/packages/playwright-core/src/utilsBundle.ts @@ -30,7 +30,7 @@ export const minimatch: typeof import('../bundles/utils/node_modules/@types/mini export const open: typeof import('../bundles/utils/node_modules/open') = require('./utilsBundleImpl').open; export const PNG: typeof import('../bundles/utils/node_modules/@types/pngjs').PNG = require('./utilsBundleImpl').PNG; export const program: typeof import('../bundles/utils/node_modules/commander').program = require('./utilsBundleImpl').program; -export const progress: typeof import('../bundles/utils/node_modules/@types/progress') = require('./utilsBundleImpl').progress; +export const multiProgress: typeof import('../bundles/utils/node_modules/@types/multi-progress') = require('./utilsBundleImpl').multiProgress; export const SocksProxyAgent: typeof import('../bundles/utils/node_modules/socks-proxy-agent').SocksProxyAgent = require('./utilsBundleImpl').SocksProxyAgent; export const yaml: typeof import('../bundles/utils/node_modules/yaml') = require('./utilsBundleImpl').yaml; export const ws: typeof import('../bundles/utils/node_modules/@types/ws') = require('./utilsBundleImpl').ws;