From 0406e45cf3bfab9488cd8f06cda640d5acf5b2cd Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Thu, 24 Aug 2023 07:33:32 -0700 Subject: [PATCH] chore: render download progress in the host process (#26666) --- packages/playwright-core/src/cli/program.ts | 1 + .../src/server/registry/browserFetcher.ts | 78 +++++++++- .../server/registry/oopDownloadBrowserMain.ts | 147 ++++++------------ 3 files changed, 127 insertions(+), 99 deletions(-) diff --git a/packages/playwright-core/src/cli/program.ts b/packages/playwright-core/src/cli/program.ts index 802fb19732..5fd3f385c4 100644 --- a/packages/playwright-core/src/cli/program.ts +++ b/packages/playwright-core/src/cli/program.ts @@ -177,6 +177,7 @@ program .description('Removes browsers used by this installation of Playwright from the system (chromium, firefox, webkit, ffmpeg). This does not include branded channels.') .option('--all', 'Removes all browsers used by any Playwright installation from the system.') .action(async (options: { all?: boolean }) => { + delete process.env.PLAYWRIGHT_SKIP_BROWSER_GC; await registry.uninstall(!!options.all).then(({ numberOfBrowsersLeft }) => { if (!options.all && numberOfBrowsersLeft > 0) { console.log('Successfully uninstalled Playwright browsers for the current Playwright installation.'); diff --git a/packages/playwright-core/src/server/registry/browserFetcher.ts b/packages/playwright-core/src/server/registry/browserFetcher.ts index 23f924dc7a..9dfc6dc448 100644 --- a/packages/playwright-core/src/server/registry/browserFetcher.ts +++ b/packages/playwright-core/src/server/registry/browserFetcher.ts @@ -22,8 +22,10 @@ import childProcess from 'child_process'; import { existsAsync } from '../../utils/fileUtils'; import { debugLogger } from '../../common/debugLogger'; import { ManualPromise } from '../../utils/manualPromise'; -import { colors } from '../../utilsBundle'; +import { colors, progress as ProgressBar } from '../../utilsBundle'; import { browserDirectoryToMarkerFilePath } from '.'; +import { getUserAgent } from '../../utils/userAgent'; +import type { DownloadParams } from './oopDownloadBrowserMain'; export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string | undefined, downloadURLs: string[], downloadFileName: string, downloadConnectionTimeout: number): Promise { if (await existsAsync(browserDirectoryToMarkerFilePath(browserDirectory))) { @@ -70,12 +72,15 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec * Thats why we execute it in a separate process and check manually if the destination file exists. * https://github.com/microsoft/playwright/issues/17394 */ -function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirectory: string, url: string, zipPath: string, executablePath: string | undefined, downloadConnectionTimeout: number): Promise<{ error: Error | null }> { - const cp = childProcess.fork(path.join(__dirname, 'oopDownloadBrowserMain.js'), [title, browserDirectory, url, zipPath, executablePath || '', String(downloadConnectionTimeout)]); +function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirectory: string, url: string, zipPath: string, executablePath: string | undefined, connectionTimeout: number): Promise<{ error: Error | null }> { + const cp = childProcess.fork(path.join(__dirname, 'oopDownloadBrowserMain.js')); const promise = new ManualPromise<{ error: Error | null }>(); + const progress = getDownloadProgress(); cp.on('message', (message: any) => { if (message?.method === 'log') debugLogger.log('install', message.params.message); + if (message?.method === 'progress') + progress(message.params.done, message.params.total); }); cp.on('exit', code => { if (code !== 0) { @@ -90,6 +95,20 @@ function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirect cp.on('error', error => { promise.resolve({ error }); }); + + debugLogger.log('install', `running download:`); + debugLogger.log('install', `-- from url: ${url}`); + debugLogger.log('install', `-- to location: ${zipPath}`); + const downloadParams: DownloadParams = { + title, + browserDirectory, + url, + zipPath, + executablePath, + connectionTimeout, + userAgent: getUserAgent(), + }; + cp.send({ method: 'download', params: downloadParams }); return promise; } @@ -100,3 +119,56 @@ export function logPolitely(toBeLogged: string) { if (!logLevelDisplay) console.log(toBeLogged); // eslint-disable-line no-console } + +type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; + +function getDownloadProgress(): OnProgressCallback { + if (process.stdout.isTTY) + return getAnimatedDownloadProgress(); + return getBasicDownloadProgress(); +} + +function getAnimatedDownloadProgress(): OnProgressCallback { + let progressBar: ProgressBar; + let lastDownloadedBytes = 0; + + return (downloadedBytes: number, totalBytes: number) => { + if (!progressBar) { + progressBar = new ProgressBar( + `${toMegabytes( + totalBytes + )} [:bar] :percent :etas`, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + }; +} + +function getBasicDownloadProgress(): OnProgressCallback { + const totalRows = 10; + const stepWidth = 8; + let lastRow = -1; + return (downloadedBytes: number, totalBytes: number) => { + const percentage = downloadedBytes / totalBytes; + const row = Math.floor(totalRows * percentage); + if (row > lastRow) { + lastRow = row; + const percentageString = String(percentage * 100 | 0).padStart(3); + // eslint-disable-next-line no-console + console.log(`|${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`); + } + }; +} + +function toMegabytes(bytes: number) { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; +} diff --git a/packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts b/packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts index 9b379c668a..b0332405c2 100644 --- a/packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts +++ b/packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts @@ -15,37 +15,41 @@ */ import fs from 'fs'; -import { progress as ProgressBar } from '../../utilsBundle'; +import path from 'path'; import { httpRequest } from '../../utils/network'; import { ManualPromise } from '../../utils/manualPromise'; import { extract } from '../../zipBundle'; -import { getUserAgent } from '../../utils/userAgent'; -import { browserDirectoryToMarkerFilePath } from '.'; -type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; -type DownloadFileLogger = (message: string) => void; -type DownloadFileOptions = { - progressCallback: OnProgressCallback, - log: DownloadFileLogger, - userAgent: string, - connectionTimeout: number, +export type DownloadParams = { + title: string; + browserDirectory: string; + url: string; + zipPath: string; + executablePath: string | undefined; + connectionTimeout: number; + userAgent: string; }; -function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions): Promise { - const { - progressCallback, - log = () => { }, - } = options; - log(`running download:`); - log(`-- from url: ${url}`); - log(`-- to location: ${destinationPath}`); +function log(message: string) { + process.send?.({ method: 'log', params: { message } }); +} + +function progress(done: number, total: number) { + process.send?.({ method: 'progress', params: { done, total } }); +} + +function browserDirectoryToMarkerFilePath(browserDirectory: string): string { + return path.join(browserDirectory, 'INSTALLATION_COMPLETE'); +} + +function downloadFile(options: DownloadParams): Promise { let downloadedBytes = 0; let totalBytes = 0; const promise = new ManualPromise(); httpRequest({ - url, + url: options.url, headers: { 'User-Agent': options.userAgent, }, @@ -55,7 +59,7 @@ function downloadFile(url: string, destinationPath: string, options: DownloadFil if (response.statusCode !== 200) { let content = ''; const handleError = () => { - const error = new Error(`Download failed: server returned code ${response.statusCode} body '${content}'. URL: ${url}`); + const error = new Error(`Download failed: server returned code ${response.statusCode} body '${content}'. URL: ${options.url}`); // consume response data to free up memory response.resume(); promise.reject(error); @@ -68,11 +72,11 @@ function downloadFile(url: string, destinationPath: string, options: DownloadFil } totalBytes = parseInt(response.headers['content-length'] || '0', 10); log(`-- total bytes: ${totalBytes}`); - const file = fs.createWriteStream(destinationPath); + const file = fs.createWriteStream(options.zipPath); file.on('finish', () => { if (downloadedBytes !== totalBytes) { log(`-- download failed, size mismatch: ${downloadedBytes} != ${totalBytes}`); - promise.reject(new Error(`Download failed: size mismatch, file size: ${downloadedBytes}, expected size: ${totalBytes} URL: ${url}`)); + promise.reject(new Error(`Download failed: size mismatch, file size: ${downloadedBytes}, expected size: ${totalBytes} URL: ${options.url}`)); } else { log(`-- download complete, size: ${downloadedBytes}`); promise.resolve(); @@ -86,85 +90,36 @@ function downloadFile(url: string, destinationPath: string, options: DownloadFil function onData(chunk: string) { downloadedBytes += chunk.length; - progressCallback!(downloadedBytes, totalBytes); + progress(downloadedBytes, totalBytes); } } -function getDownloadProgress(): OnProgressCallback { - if (process.stdout.isTTY) - return getAnimatedDownloadProgress(); - return getBasicDownloadProgress(); -} - -function getAnimatedDownloadProgress(): OnProgressCallback { - let progressBar: ProgressBar; - let lastDownloadedBytes = 0; - - return (downloadedBytes: number, totalBytes: number) => { - if (!progressBar) { - progressBar = new ProgressBar( - `${toMegabytes( - totalBytes - )} [:bar] :percent :etas`, - { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, - } - ); - } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); - }; -} - -function getBasicDownloadProgress(): OnProgressCallback { - const totalRows = 10; - const stepWidth = 8; - let lastRow = -1; - return (downloadedBytes: number, totalBytes: number) => { - const percentage = downloadedBytes / totalBytes; - const row = Math.floor(totalRows * percentage); - if (row > lastRow) { - lastRow = row; - const percentageString = String(percentage * 100 | 0).padStart(3); - // eslint-disable-next-line no-console - console.log(`|${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`); - } - }; -} - -function toMegabytes(bytes: number) { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; -} - -async function main() { - const log = (message: string) => process.send?.({ method: 'log', params: { message } }); - const [title, browserDirectory, url, zipPath, executablePath, downloadConnectionTimeout] = process.argv.slice(2); - await downloadFile(url, zipPath, { - progressCallback: getDownloadProgress(), - userAgent: getUserAgent(), - log, - connectionTimeout: +downloadConnectionTimeout, - }); - log(`SUCCESS downloading ${title}`); +async function main(options: DownloadParams) { + await downloadFile(options); log(`extracting archive`); - log(`-- zip: ${zipPath}`); - log(`-- location: ${browserDirectory}`); - await extract(zipPath, { dir: browserDirectory }); - if (executablePath) { - log(`fixing permissions at ${executablePath}`); - await fs.promises.chmod(executablePath, 0o755); + await extract(options.zipPath, { dir: options.browserDirectory }); + if (options.executablePath) { + log(`fixing permissions at ${options.executablePath}`); + await fs.promises.chmod(options.executablePath, 0o755); } - await fs.promises.writeFile(browserDirectoryToMarkerFilePath(browserDirectory), ''); + await fs.promises.writeFile(browserDirectoryToMarkerFilePath(options.browserDirectory), ''); } -main().catch(error => { - // eslint-disable-next-line no-console - console.error(error); - // eslint-disable-next-line no-restricted-properties - process.exit(1); +process.on('message', async message => { + const { method, params } = message as any; + if (method === 'download') { + try { + await main(params); + // eslint-disable-next-line no-restricted-properties + process.exit(0); + } catch (e) { + // eslint-disable-next-line no-console + console.error(e); + // eslint-disable-next-line no-restricted-properties + process.exit(1); + } + } }); + +// eslint-disable-next-line no-restricted-properties +process.on('disconnect', () => { process.exit(0); });