Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore: render download progress in the host process (#26666)

Browse source
This commit is contained in:
Pavel Feldman 2023-08-24 07:33:32 -07:00 committed by GitHub
parent 697429d222
commit 0406e45cf3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 127 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
packages/playwright-core/src/cli/program.ts
View file

@ -177,6 +177,7 @@ program
.description('Removes browsers used by this installation of Playwright from the system (chromium, firefox, webkit, ffmpeg). This does not include branded channels.')
.option('--all', 'Removes all browsers used by any Playwright installation from the system.')
.action(async (options: { all?: boolean }) => {
delete process.env.PLAYWRIGHT_SKIP_BROWSER_GC;
await registry.uninstall(!!options.all).then(({ numberOfBrowsersLeft }) => {
if (!options.all && numberOfBrowsersLeft > 0) {
console.log('Successfully uninstalled Playwright browsers for the current Playwright installation.');

78
packages/playwright-core/src/server/registry/browserFetcher.ts
View file

@ -22,8 +22,10 @@ import childProcess from 'child_process';
import { existsAsync } from '../../utils/fileUtils';
import { debugLogger } from '../../common/debugLogger';
import { ManualPromise } from '../../utils/manualPromise';
import { colors } from '../../utilsBundle';
import { colors, progress as ProgressBar } from '../../utilsBundle';
import { browserDirectoryToMarkerFilePath } from '.';
import { getUserAgent } from '../../utils/userAgent';
import type { DownloadParams } from './oopDownloadBrowserMain';
export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string | undefined, downloadURLs: string[], downloadFileName: string, downloadConnectionTimeout: number): Promise<boolean> {
if (await existsAsync(browserDirectoryToMarkerFilePath(browserDirectory))) {
@ -70,12 +72,15 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec
* Thats why we execute it in a separate process and check manually if the destination file exists.
* https://github.com/microsoft/playwright/issues/17394
*/
function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirectory: string, url: string, zipPath: string, executablePath: string | undefined, downloadConnectionTimeout: number): Promise<{ error: Error | null }> {
const cp = childProcess.fork(path.join(__dirname, 'oopDownloadBrowserMain.js'), [title, browserDirectory, url, zipPath, executablePath || '', String(downloadConnectionTimeout)]);
function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirectory: string, url: string, zipPath: string, executablePath: string | undefined, connectionTimeout: number): Promise<{ error: Error | null }> {
const cp = childProcess.fork(path.join(__dirname, 'oopDownloadBrowserMain.js'));
const promise = new ManualPromise<{ error: Error | null }>();
const progress = getDownloadProgress();
cp.on('message', (message: any) => {
if (message?.method === 'log')
debugLogger.log('install', message.params.message);
if (message?.method === 'progress')
progress(message.params.done, message.params.total);
});
cp.on('exit', code => {
if (code !== 0) {
@ -90,6 +95,20 @@ function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirect
cp.on('error', error => {
promise.resolve({ error });
});
debugLogger.log('install', `running download:`);
debugLogger.log('install', `-- from url: ${url}`);
debugLogger.log('install', `-- to location: ${zipPath}`);
const downloadParams: DownloadParams = {
title,
browserDirectory,
url,
zipPath,
executablePath,
connectionTimeout,
userAgent: getUserAgent(),
};
cp.send({ method: 'download', params: downloadParams });
return promise;
}
@ -100,3 +119,56 @@ export function logPolitely(toBeLogged: string) {
if (!logLevelDisplay)
console.log(toBeLogged); // eslint-disable-line no-console
}
type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
function getDownloadProgress(): OnProgressCallback {
if (process.stdout.isTTY)
return getAnimatedDownloadProgress();
return getBasicDownloadProgress();
}
function getAnimatedDownloadProgress(): OnProgressCallback {
let progressBar: ProgressBar;
let lastDownloadedBytes = 0;
return (downloadedBytes: number, totalBytes: number) => {
if (!progressBar) {
progressBar = new ProgressBar(
`${toMegabytes(
totalBytes
)} [:bar] :percent :etas`,
{
complete: '=',
incomplete: ' ',
width: 20,
total: totalBytes,
}
);
}
const delta = downloadedBytes - lastDownloadedBytes;
lastDownloadedBytes = downloadedBytes;
progressBar.tick(delta);
};
}
function getBasicDownloadProgress(): OnProgressCallback {
const totalRows = 10;
const stepWidth = 8;
let lastRow = -1;
return (downloadedBytes: number, totalBytes: number) => {
const percentage = downloadedBytes / totalBytes;
const row = Math.floor(totalRows * percentage);
if (row > lastRow) {
lastRow = row;
const percentageString = String(percentage * 100 | 0).padStart(3);
// eslint-disable-next-line no-console
console.log(`|${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`);
}
};
}
function toMegabytes(bytes: number) {
const mb = bytes / 1024 / 1024;
return `${Math.round(mb * 10) / 10} Mb`;
}

147
packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts
View file

@ -15,37 +15,41 @@
*/
import fs from 'fs';
import { progress as ProgressBar } from '../../utilsBundle';
import path from 'path';
import { httpRequest } from '../../utils/network';
import { ManualPromise } from '../../utils/manualPromise';
import { extract } from '../../zipBundle';
import { getUserAgent } from '../../utils/userAgent';
import { browserDirectoryToMarkerFilePath } from '.';
type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
type DownloadFileLogger = (message: string) => void;
type DownloadFileOptions = {
progressCallback: OnProgressCallback,
log: DownloadFileLogger,
userAgent: string,
connectionTimeout: number,
export type DownloadParams = {
title: string;
browserDirectory: string;
url: string;
zipPath: string;
executablePath: string | undefined;
connectionTimeout: number;
userAgent: string;
};
function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions): Promise<void> {
const {
progressCallback,
log = () => { },
} = options;
log(`running download:`);
log(`-- from url: ${url}`);
log(`-- to location: ${destinationPath}`);
function log(message: string) {
process.send?.({ method: 'log', params: { message } });
}
function progress(done: number, total: number) {
process.send?.({ method: 'progress', params: { done, total } });
}
function browserDirectoryToMarkerFilePath(browserDirectory: string): string {
return path.join(browserDirectory, 'INSTALLATION_COMPLETE');
}
function downloadFile(options: DownloadParams): Promise<void> {
let downloadedBytes = 0;
let totalBytes = 0;
const promise = new ManualPromise<void>();
httpRequest({
url,
url: options.url,
headers: {
'User-Agent': options.userAgent,
},
@ -55,7 +59,7 @@ function downloadFile(url: string, destinationPath: string, options: DownloadFil
if (response.statusCode !== 200) {
let content = '';
const handleError = () => {
const error = new Error(`Download failed: server returned code ${response.statusCode} body '${content}'. URL: ${url}`);
const error = new Error(`Download failed: server returned code ${response.statusCode} body '${content}'. URL: ${options.url}`);
// consume response data to free up memory
response.resume();
promise.reject(error);
@ -68,11 +72,11 @@ function downloadFile(url: string, destinationPath: string, options: DownloadFil
}
totalBytes = parseInt(response.headers['content-length'] || '0', 10);
log(`-- total bytes: ${totalBytes}`);
const file = fs.createWriteStream(destinationPath);
const file = fs.createWriteStream(options.zipPath);
file.on('finish', () => {
if (downloadedBytes !== totalBytes) {
log(`-- download failed, size mismatch: ${downloadedBytes} != ${totalBytes}`);
promise.reject(new Error(`Download failed: size mismatch, file size: ${downloadedBytes}, expected size: ${totalBytes} URL: ${url}`));
promise.reject(new Error(`Download failed: size mismatch, file size: ${downloadedBytes}, expected size: ${totalBytes} URL: ${options.url}`));
} else {
log(`-- download complete, size: ${downloadedBytes}`);
promise.resolve();
@ -86,85 +90,36 @@ function downloadFile(url: string, destinationPath: string, options: DownloadFil
function onData(chunk: string) {
downloadedBytes += chunk.length;
progressCallback!(downloadedBytes, totalBytes);
progress(downloadedBytes, totalBytes);
}
}
function getDownloadProgress(): OnProgressCallback {
if (process.stdout.isTTY)
return getAnimatedDownloadProgress();
return getBasicDownloadProgress();
}
function getAnimatedDownloadProgress(): OnProgressCallback {
let progressBar: ProgressBar;
let lastDownloadedBytes = 0;
return (downloadedBytes: number, totalBytes: number) => {
if (!progressBar) {
progressBar = new ProgressBar(
`${toMegabytes(
totalBytes
)} [:bar] :percent :etas`,
{
complete: '=',
incomplete: ' ',
width: 20,
total: totalBytes,
}
);
}
const delta = downloadedBytes - lastDownloadedBytes;
lastDownloadedBytes = downloadedBytes;
progressBar.tick(delta);
};
}
function getBasicDownloadProgress(): OnProgressCallback {
const totalRows = 10;
const stepWidth = 8;
let lastRow = -1;
return (downloadedBytes: number, totalBytes: number) => {
const percentage = downloadedBytes / totalBytes;
const row = Math.floor(totalRows * percentage);
if (row > lastRow) {
lastRow = row;
const percentageString = String(percentage * 100 | 0).padStart(3);
// eslint-disable-next-line no-console
console.log(`|${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`);
}
};
}
function toMegabytes(bytes: number) {
const mb = bytes / 1024 / 1024;
return `${Math.round(mb * 10) / 10} Mb`;
}
async function main() {
const log = (message: string) => process.send?.({ method: 'log', params: { message } });
const [title, browserDirectory, url, zipPath, executablePath, downloadConnectionTimeout] = process.argv.slice(2);
await downloadFile(url, zipPath, {
progressCallback: getDownloadProgress(),
userAgent: getUserAgent(),
log,
connectionTimeout: +downloadConnectionTimeout,
});
log(`SUCCESS downloading ${title}`);
async function main(options: DownloadParams) {
await downloadFile(options);
log(`extracting archive`);
log(`-- zip: ${zipPath}`);
log(`-- location: ${browserDirectory}`);
await extract(zipPath, { dir: browserDirectory });
if (executablePath) {
log(`fixing permissions at ${executablePath}`);
await fs.promises.chmod(executablePath, 0o755);
await extract(options.zipPath, { dir: options.browserDirectory });
if (options.executablePath) {
log(`fixing permissions at ${options.executablePath}`);
await fs.promises.chmod(options.executablePath, 0o755);
}
await fs.promises.writeFile(browserDirectoryToMarkerFilePath(browserDirectory), '');
await fs.promises.writeFile(browserDirectoryToMarkerFilePath(options.browserDirectory), '');
}
main().catch(error => {
// eslint-disable-next-line no-console
console.error(error);
// eslint-disable-next-line no-restricted-properties
process.exit(1);
process.on('message', async message => {
const { method, params } = message as any;
if (method === 'download') {
try {
await main(params);
// eslint-disable-next-line no-restricted-properties
process.exit(0);
} catch (e) {
// eslint-disable-next-line no-console
console.error(e);
// eslint-disable-next-line no-restricted-properties
process.exit(1);
}
}
});
// eslint-disable-next-line no-restricted-properties
process.on('disconnect', () => { process.exit(0); });