chore: download browsers out of process (#17743)
https://github.com/microsoft/playwright/issues/17394
This commit is contained in:
Max Schmitt
GitHub
parent
9b35a8071f
commit
4306b38c2e
|
|
@ -15,68 +15,43 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import { progress as ProgressBar } from '../../utilsBundle';
|
import path from 'path';
|
||||||
import { httpRequest } from '../../common/netUtils';
|
import childProcess from 'child_process';
|
||||||
|
import { ManualPromise } from '../../utils/manualPromise';
|
||||||
|
|
||||||
type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
|
|
||||||
type DownloadFileLogger = (message: string) => void;
|
type DownloadFileLogger = (message: string) => void;
|
||||||
type DownloadFileOptions = {
|
type DownloadFileOptions = {
|
||||||
progressCallback?: OnProgressCallback,
|
progressBarName?: string,
|
||||||
log?: DownloadFileLogger,
|
log?: DownloadFileLogger,
|
||||||
userAgent?: string
|
userAgent?: string
|
||||||
};
|
};
|
||||||
|
|
||||||
function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{ error: any }> {
|
/**
|
||||||
const {
|
* Node.js has a bug where the process can exit with 0 code even though there was an uncaught exception.
|
||||||
progressCallback,
|
* Thats why we execute it in a separate process and check manually if the destination file exists.
|
||||||
log = () => { },
|
* https://github.com/microsoft/playwright/issues/17394
|
||||||
} = options;
|
*/
|
||||||
log(`running download:`);
|
function downloadFileOutOfProcess(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{ error: Error | null }> {
|
||||||
log(`-- from url: ${url}`);
|
const cp = childProcess.fork(path.join(__dirname, 'oopDownloadMain.js'), [url, destinationPath, options.progressBarName || '', options.userAgent || '']);
|
||||||
log(`-- to location: ${destinationPath}`);
|
const promise = new ManualPromise<{ error: Error | null }>();
|
||||||
let fulfill: ({ error }: { error: any }) => void = ({ error }) => { };
|
cp.on('message', (message: any) => {
|
||||||
let downloadedBytes = 0;
|
if (message?.method === 'log')
|
||||||
let totalBytes = 0;
|
options.log?.(message.params.message);
|
||||||
|
});
|
||||||
const promise: Promise<{ error: any }> = new Promise(x => { fulfill = x; });
|
cp.on('exit', code => {
|
||||||
|
if (code !== 0) {
|
||||||
httpRequest({
|
promise.resolve({ error: new Error(`Download failure, code=${code}`) });
|
||||||
url,
|
|
||||||
headers: options.userAgent ? {
|
|
||||||
'User-Agent': options.userAgent,
|
|
||||||
} : undefined,
|
|
||||||
timeout: 10_000,
|
|
||||||
}, response => {
|
|
||||||
log(`-- response status code: ${response.statusCode}`);
|
|
||||||
if (response.statusCode !== 200) {
|
|
||||||
let content = '';
|
|
||||||
const handleError = () => {
|
|
||||||
const error = new Error(`Download failed: server returned code ${response.statusCode} body '${content}'. URL: ${url}`);
|
|
||||||
// consume response data to free up memory
|
|
||||||
response.resume();
|
|
||||||
fulfill({ error });
|
|
||||||
};
|
|
||||||
response
|
|
||||||
.on('data', chunk => content += chunk)
|
|
||||||
.on('end', handleError)
|
|
||||||
.on('error', handleError);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const file = fs.createWriteStream(destinationPath);
|
if (!fs.existsSync(destinationPath))
|
||||||
file.on('finish', () => fulfill({ error: null }));
|
promise.resolve({ error: new Error(`Download failure, ${destinationPath} does not exist`) });
|
||||||
file.on('error', error => fulfill({ error }));
|
else
|
||||||
response.pipe(file);
|
promise.resolve({ error: null });
|
||||||
totalBytes = parseInt(response.headers['content-length'] || '0', 10);
|
});
|
||||||
log(`-- total bytes: ${totalBytes}`);
|
cp.on('error', error => {
|
||||||
if (progressCallback)
|
promise.resolve({ error });
|
||||||
response.on('data', onData);
|
});
|
||||||
}, (error: any) => fulfill({ error }));
|
|
||||||
return promise;
|
return promise;
|
||||||
|
|
||||||
function onData(chunk: string) {
|
|
||||||
downloadedBytes += chunk.length;
|
|
||||||
progressCallback!(downloadedBytes, totalBytes);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type DownloadOptions = {
|
type DownloadOptions = {
|
||||||
|
|
@ -99,8 +74,9 @@ export async function download(
|
||||||
if (!Array.isArray(urls))
|
if (!Array.isArray(urls))
|
||||||
urls = [urls];
|
urls = [urls];
|
||||||
const url = urls[(attempt - 1) % urls.length];
|
const url = urls[(attempt - 1) % urls.length];
|
||||||
const { error } = await downloadFile(url, destination, {
|
|
||||||
progressCallback: getDownloadProgress(progressBarName),
|
const { error } = await downloadFileOutOfProcess(url, destination, {
|
||||||
|
progressBarName,
|
||||||
log,
|
log,
|
||||||
userAgent,
|
userAgent,
|
||||||
});
|
});
|
||||||
|
|
@ -114,56 +90,3 @@ export async function download(
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function getDownloadProgress(progressBarName: string): OnProgressCallback {
|
|
||||||
if (process.stdout.isTTY)
|
|
||||||
return _getAnimatedDownloadProgress(progressBarName);
|
|
||||||
return _getBasicDownloadProgress(progressBarName);
|
|
||||||
}
|
|
||||||
|
|
||||||
function _getAnimatedDownloadProgress(progressBarName: string): OnProgressCallback {
|
|
||||||
let progressBar: ProgressBar;
|
|
||||||
let lastDownloadedBytes = 0;
|
|
||||||
|
|
||||||
return (downloadedBytes: number, totalBytes: number) => {
|
|
||||||
if (!progressBar) {
|
|
||||||
progressBar = new ProgressBar(
|
|
||||||
`Downloading ${progressBarName} - ${toMegabytes(
|
|
||||||
totalBytes
|
|
||||||
)} [:bar] :percent :etas `,
|
|
||||||
{
|
|
||||||
complete: '=',
|
|
||||||
incomplete: ' ',
|
|
||||||
width: 20,
|
|
||||||
total: totalBytes,
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const delta = downloadedBytes - lastDownloadedBytes;
|
|
||||||
lastDownloadedBytes = downloadedBytes;
|
|
||||||
progressBar.tick(delta);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function _getBasicDownloadProgress(progressBarName: string): OnProgressCallback {
|
|
||||||
// eslint-disable-next-line no-console
|
|
||||||
console.log(`Downloading ${progressBarName}...`);
|
|
||||||
const totalRows = 10;
|
|
||||||
const stepWidth = 8;
|
|
||||||
let lastRow = -1;
|
|
||||||
return (downloadedBytes: number, totalBytes: number) => {
|
|
||||||
const percentage = downloadedBytes / totalBytes;
|
|
||||||
const row = Math.floor(totalRows * percentage);
|
|
||||||
if (row > lastRow) {
|
|
||||||
lastRow = row;
|
|
||||||
const percentageString = String(percentage * 100 | 0).padStart(3);
|
|
||||||
// eslint-disable-next-line no-console
|
|
||||||
console.log(`|${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function toMegabytes(bytes: number) {
|
|
||||||
const mb = bytes / 1024 / 1024;
|
|
||||||
return `${Math.round(mb * 10) / 10} Mb`;
|
|
||||||
}
|
|
||||||
|
|
|
||||||
148
packages/playwright-core/src/server/registry/oopDownloadMain.ts
Normal file
148
packages/playwright-core/src/server/registry/oopDownloadMain.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
||||||
|
/**
|
||||||
|
* Copyright (c) Microsoft Corporation.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import fs from 'fs';
|
||||||
|
import { progress as ProgressBar } from '../../utilsBundle';
|
||||||
|
import { httpRequest } from '../../common/netUtils';
|
||||||
|
import { ManualPromise } from '../../utils/manualPromise';
|
||||||
|
|
||||||
|
type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
|
||||||
|
type DownloadFileLogger = (message: string) => void;
|
||||||
|
type DownloadFileOptions = {
|
||||||
|
progressCallback?: OnProgressCallback,
|
||||||
|
log?: DownloadFileLogger,
|
||||||
|
userAgent?: string
|
||||||
|
};
|
||||||
|
|
||||||
|
function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<void> {
|
||||||
|
const {
|
||||||
|
progressCallback,
|
||||||
|
log = () => { },
|
||||||
|
} = options;
|
||||||
|
log(`running download:`);
|
||||||
|
log(`-- from url: ${url}`);
|
||||||
|
log(`-- to location: ${destinationPath}`);
|
||||||
|
let downloadedBytes = 0;
|
||||||
|
let totalBytes = 0;
|
||||||
|
|
||||||
|
const promise = new ManualPromise<void>();
|
||||||
|
|
||||||
|
httpRequest({
|
||||||
|
url,
|
||||||
|
headers: options.userAgent ? {
|
||||||
|
'User-Agent': options.userAgent,
|
||||||
|
} : undefined,
|
||||||
|
timeout: 10_000,
|
||||||
|
}, response => {
|
||||||
|
log(`-- response status code: ${response.statusCode}`);
|
||||||
|
if (response.statusCode !== 200) {
|
||||||
|
let content = '';
|
||||||
|
const handleError = () => {
|
||||||
|
const error = new Error(`Download failed: server returned code ${response.statusCode} body '${content}'. URL: ${url}`);
|
||||||
|
// consume response data to free up memory
|
||||||
|
response.resume();
|
||||||
|
promise.reject(error);
|
||||||
|
};
|
||||||
|
response
|
||||||
|
.on('data', chunk => content += chunk)
|
||||||
|
.on('end', handleError)
|
||||||
|
.on('error', handleError);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const file = fs.createWriteStream(destinationPath);
|
||||||
|
file.on('finish', () => promise.resolve());
|
||||||
|
file.on('error', error => promise.reject(error));
|
||||||
|
response.pipe(file);
|
||||||
|
totalBytes = parseInt(response.headers['content-length'] || '0', 10);
|
||||||
|
log(`-- total bytes: ${totalBytes}`);
|
||||||
|
if (progressCallback)
|
||||||
|
response.on('data', onData);
|
||||||
|
}, (error: any) => promise.reject(error));
|
||||||
|
return promise;
|
||||||
|
|
||||||
|
function onData(chunk: string) {
|
||||||
|
downloadedBytes += chunk.length;
|
||||||
|
progressCallback!(downloadedBytes, totalBytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDownloadProgress(progressBarName: string): OnProgressCallback {
|
||||||
|
if (process.stdout.isTTY)
|
||||||
|
return _getAnimatedDownloadProgress(progressBarName);
|
||||||
|
return _getBasicDownloadProgress(progressBarName);
|
||||||
|
}
|
||||||
|
|
||||||
|
function _getAnimatedDownloadProgress(progressBarName: string): OnProgressCallback {
|
||||||
|
let progressBar: ProgressBar;
|
||||||
|
let lastDownloadedBytes = 0;
|
||||||
|
|
||||||
|
return (downloadedBytes: number, totalBytes: number) => {
|
||||||
|
if (!progressBar) {
|
||||||
|
progressBar = new ProgressBar(
|
||||||
|
`Downloading ${progressBarName} - ${toMegabytes(
|
||||||
|
totalBytes
|
||||||
|
)} [:bar] :percent :etas `,
|
||||||
|
{
|
||||||
|
complete: '=',
|
||||||
|
incomplete: ' ',
|
||||||
|
width: 20,
|
||||||
|
total: totalBytes,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const delta = downloadedBytes - lastDownloadedBytes;
|
||||||
|
lastDownloadedBytes = downloadedBytes;
|
||||||
|
progressBar.tick(delta);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function _getBasicDownloadProgress(progressBarName: string): OnProgressCallback {
|
||||||
|
// eslint-disable-next-line no-console
|
||||||
|
console.log(`Downloading ${progressBarName}...`);
|
||||||
|
const totalRows = 10;
|
||||||
|
const stepWidth = 8;
|
||||||
|
let lastRow = -1;
|
||||||
|
return (downloadedBytes: number, totalBytes: number) => {
|
||||||
|
const percentage = downloadedBytes / totalBytes;
|
||||||
|
const row = Math.floor(totalRows * percentage);
|
||||||
|
if (row > lastRow) {
|
||||||
|
lastRow = row;
|
||||||
|
const percentageString = String(percentage * 100 | 0).padStart(3);
|
||||||
|
// eslint-disable-next-line no-console
|
||||||
|
console.log(`|${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function toMegabytes(bytes: number) {
|
||||||
|
const mb = bytes / 1024 / 1024;
|
||||||
|
return `${Math.round(mb * 10) / 10} Mb`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const [url, destination, progressBarName, userAgent] = process.argv.slice(2);
|
||||||
|
await downloadFile(url, destination, {
|
||||||
|
progressCallback: getDownloadProgress(progressBarName),
|
||||||
|
userAgent,
|
||||||
|
log: message => process.send?.({ method: 'log', params: { message } }),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(error => {
|
||||||
|
// eslint-disable-next-line no-console
|
||||||
|
console.error(error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
Loading…
Reference in a new issue