allow parallel downloads

This commit is contained in:
Simon Knott 2024-11-29 11:55:31 +01:00
parent 7eaf37d71d
commit 1f05f90bcd
No known key found for this signature in database
GPG key ID: 8CEDC00028084AEC
5 changed files with 58 additions and 18 deletions

View file

@ -18,9 +18,9 @@
"jpeg-js": "0.4.4", "jpeg-js": "0.4.4",
"mime": "^3.0.0", "mime": "^3.0.0",
"minimatch": "^3.1.2", "minimatch": "^3.1.2",
"multi-progress": "^4.0.0",
"open": "8.4.0", "open": "8.4.0",
"pngjs": "6.0.0", "pngjs": "6.0.0",
"progress": "2.0.3",
"proxy-from-env": "1.1.0", "proxy-from-env": "1.1.0",
"retry": "0.12.0", "retry": "0.12.0",
"signal-exit": "3.0.7", "signal-exit": "3.0.7",
@ -34,8 +34,8 @@
"@types/diff": "^6.0.0", "@types/diff": "^6.0.0",
"@types/mime": "^2.0.3", "@types/mime": "^2.0.3",
"@types/minimatch": "^3.0.5", "@types/minimatch": "^3.0.5",
"@types/multi-progress": "^2.0.6",
"@types/pngjs": "^6.0.1", "@types/pngjs": "^6.0.1",
"@types/progress": "^2.0.5",
"@types/proper-lockfile": "^4.1.2", "@types/proper-lockfile": "^4.1.2",
"@types/proxy-from-env": "^1.0.1", "@types/proxy-from-env": "^1.0.1",
"@types/stack-utils": "^2.0.1", "@types/stack-utils": "^2.0.1",
@ -76,6 +76,17 @@
"integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==", "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==",
"dev": true "dev": true
}, },
"node_modules/@types/multi-progress": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/@types/multi-progress/-/multi-progress-2.0.6.tgz",
"integrity": "sha512-S+DW6NQdvwJ6liFUmGNbGtUwFwHm/qx2D2hrlX5HY5f55hRpteS1F1pHfrC2AoCvPNAeVuFVlZTDyIkPssMujg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*",
"@types/progress": "*"
}
},
"node_modules/@types/node": { "node_modules/@types/node": {
"version": "17.0.25", "version": "17.0.25",
"resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.25.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.25.tgz",
@ -330,6 +341,15 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}, },
"node_modules/multi-progress": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/multi-progress/-/multi-progress-4.0.0.tgz",
"integrity": "sha512-9zcjyOou3FFCKPXsmkbC3ethv51SFPoA4dJD6TscIp2pUmy26kBDZW6h9XofPELrzseSkuD7r0V+emGEeo39Pg==",
"license": "MIT",
"peerDependencies": {
"progress": "^2.0.0"
}
},
"node_modules/open": { "node_modules/open": {
"version": "8.4.0", "version": "8.4.0",
"resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz", "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz",
@ -358,6 +378,7 @@
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
"peer": true,
"engines": { "engines": {
"node": ">=0.4.0" "node": ">=0.4.0"
} }
@ -498,6 +519,16 @@
"integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==", "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==",
"dev": true "dev": true
}, },
"@types/multi-progress": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/@types/multi-progress/-/multi-progress-2.0.6.tgz",
"integrity": "sha512-S+DW6NQdvwJ6liFUmGNbGtUwFwHm/qx2D2hrlX5HY5f55hRpteS1F1pHfrC2AoCvPNAeVuFVlZTDyIkPssMujg==",
"dev": true,
"requires": {
"@types/node": "*",
"@types/progress": "*"
}
},
"@types/node": { "@types/node": {
"version": "17.0.25", "version": "17.0.25",
"resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.25.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.25.tgz",
@ -690,6 +721,12 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}, },
"multi-progress": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/multi-progress/-/multi-progress-4.0.0.tgz",
"integrity": "sha512-9zcjyOou3FFCKPXsmkbC3ethv51SFPoA4dJD6TscIp2pUmy26kBDZW6h9XofPELrzseSkuD7r0V+emGEeo39Pg==",
"requires": {}
},
"open": { "open": {
"version": "8.4.0", "version": "8.4.0",
"resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz", "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz",
@ -708,7 +745,8 @@
"progress": { "progress": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
"peer": true
}, },
"proxy-from-env": { "proxy-from-env": {
"version": "1.1.0", "version": "1.1.0",

View file

@ -19,9 +19,9 @@
"jpeg-js": "0.4.4", "jpeg-js": "0.4.4",
"mime": "^3.0.0", "mime": "^3.0.0",
"minimatch": "^3.1.2", "minimatch": "^3.1.2",
"multi-progress": "^4.0.0",
"open": "8.4.0", "open": "8.4.0",
"pngjs": "6.0.0", "pngjs": "6.0.0",
"progress": "2.0.3",
"proxy-from-env": "1.1.0", "proxy-from-env": "1.1.0",
"retry": "0.12.0", "retry": "0.12.0",
"signal-exit": "3.0.7", "signal-exit": "3.0.7",
@ -35,8 +35,8 @@
"@types/diff": "^6.0.0", "@types/diff": "^6.0.0",
"@types/mime": "^2.0.3", "@types/mime": "^2.0.3",
"@types/minimatch": "^3.0.5", "@types/minimatch": "^3.0.5",
"@types/multi-progress": "^2.0.6",
"@types/pngjs": "^6.0.1", "@types/pngjs": "^6.0.1",
"@types/progress": "^2.0.5",
"@types/proper-lockfile": "^4.1.2", "@types/proper-lockfile": "^4.1.2",
"@types/proxy-from-env": "^1.0.1", "@types/proxy-from-env": "^1.0.1",
"@types/stack-utils": "^2.0.1", "@types/stack-utils": "^2.0.1",

View file

@ -49,8 +49,8 @@ export { PNG } from 'pngjs';
export { program } from 'commander'; export { program } from 'commander';
import progressLibrary from 'progress'; import multiProgressLibrary from 'multi-progress';
export const progress = progressLibrary; export const multiProgress = multiProgressLibrary;
export { SocksProxyAgent } from 'socks-proxy-agent'; export { SocksProxyAgent } from 'socks-proxy-agent';

View file

@ -22,7 +22,7 @@ import childProcess from 'child_process';
import { existsAsync } from '../../utils/fileUtils'; import { existsAsync } from '../../utils/fileUtils';
import { debugLogger } from '../../utils/debugLogger'; import { debugLogger } from '../../utils/debugLogger';
import { ManualPromise } from '../../utils/manualPromise'; import { ManualPromise } from '../../utils/manualPromise';
import { colors, progress as ProgressBar } from '../../utilsBundle'; import { colors, multiProgress as MultiProgressBar } from '../../utilsBundle';
import { browserDirectoryToMarkerFilePath } from '.'; import { browserDirectoryToMarkerFilePath } from '.';
import { getUserAgent } from '../../utils/userAgent'; import { getUserAgent } from '../../utils/userAgent';
import type { DownloadParams } from './oopDownloadBrowserMain'; import type { DownloadParams } from './oopDownloadBrowserMain';
@ -75,7 +75,7 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec
function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirectory: string, url: string, zipPath: string, executablePath: string | undefined, connectionTimeout: number): Promise<{ error: Error | null }> { function downloadBrowserWithProgressBarOutOfProcess(title: string, browserDirectory: string, url: string, zipPath: string, executablePath: string | undefined, connectionTimeout: number): Promise<{ error: Error | null }> {
const cp = childProcess.fork(path.join(__dirname, 'oopDownloadBrowserMain.js')); const cp = childProcess.fork(path.join(__dirname, 'oopDownloadBrowserMain.js'));
const promise = new ManualPromise<{ error: Error | null }>(); const promise = new ManualPromise<{ error: Error | null }>();
const progress = getDownloadProgress(); const progress = getDownloadProgress(title);
cp.on('message', (message: any) => { cp.on('message', (message: any) => {
if (message?.method === 'log') if (message?.method === 'log')
debugLogger.log('install', message.params.message); debugLogger.log('install', message.params.message);
@ -122,20 +122,22 @@ export function logPolitely(toBeLogged: string) {
type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
function getDownloadProgress(): OnProgressCallback { function getDownloadProgress(title: string): OnProgressCallback {
if (process.stdout.isTTY) if (process.stdout.isTTY)
return getAnimatedDownloadProgress(); return getAnimatedDownloadProgress(title);
return getBasicDownloadProgress(); return getBasicDownloadProgress(title);
} }
function getAnimatedDownloadProgress(): OnProgressCallback { const multiProgress = new MultiProgressBar();
function getAnimatedDownloadProgress(title: string): OnProgressCallback {
let progressBar: ProgressBar; let progressBar: ProgressBar;
let lastDownloadedBytes = 0; let lastDownloadedBytes = 0;
return (downloadedBytes: number, totalBytes: number) => { return (downloadedBytes: number, totalBytes: number) => {
if (!progressBar) { if (!progressBar) {
progressBar = new ProgressBar( progressBar = multiProgress.newBar(
`${toMegabytes( `${title} ${toMegabytes(
totalBytes totalBytes
)} [:bar] :percent :etas`, )} [:bar] :percent :etas`,
{ {
@ -152,7 +154,7 @@ function getAnimatedDownloadProgress(): OnProgressCallback {
}; };
} }
function getBasicDownloadProgress(): OnProgressCallback { function getBasicDownloadProgress(title: string): OnProgressCallback {
const totalRows = 10; const totalRows = 10;
const stepWidth = 8; const stepWidth = 8;
let lastRow = -1; let lastRow = -1;
@ -163,7 +165,7 @@ function getBasicDownloadProgress(): OnProgressCallback {
lastRow = row; lastRow = row;
const percentageString = String(percentage * 100 | 0).padStart(3); const percentageString = String(percentage * 100 | 0).padStart(3);
// eslint-disable-next-line no-console // eslint-disable-next-line no-console
console.log(`|${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`); console.log(`${title} |${'■'.repeat(row * stepWidth)}${' '.repeat((totalRows - row) * stepWidth)}| ${percentageString}% of ${toMegabytes(totalBytes)}`);
} }
}; };
} }

View file

@ -30,7 +30,7 @@ export const minimatch: typeof import('../bundles/utils/node_modules/@types/mini
export const open: typeof import('../bundles/utils/node_modules/open') = require('./utilsBundleImpl').open; export const open: typeof import('../bundles/utils/node_modules/open') = require('./utilsBundleImpl').open;
export const PNG: typeof import('../bundles/utils/node_modules/@types/pngjs').PNG = require('./utilsBundleImpl').PNG; export const PNG: typeof import('../bundles/utils/node_modules/@types/pngjs').PNG = require('./utilsBundleImpl').PNG;
export const program: typeof import('../bundles/utils/node_modules/commander').program = require('./utilsBundleImpl').program; export const program: typeof import('../bundles/utils/node_modules/commander').program = require('./utilsBundleImpl').program;
export const progress: typeof import('../bundles/utils/node_modules/@types/progress') = require('./utilsBundleImpl').progress; export const multiProgress: typeof import('../bundles/utils/node_modules/@types/multi-progress') = require('./utilsBundleImpl').multiProgress;
export const SocksProxyAgent: typeof import('../bundles/utils/node_modules/socks-proxy-agent').SocksProxyAgent = require('./utilsBundleImpl').SocksProxyAgent; export const SocksProxyAgent: typeof import('../bundles/utils/node_modules/socks-proxy-agent').SocksProxyAgent = require('./utilsBundleImpl').SocksProxyAgent;
export const yaml: typeof import('../bundles/utils/node_modules/yaml') = require('./utilsBundleImpl').yaml; export const yaml: typeof import('../bundles/utils/node_modules/yaml') = require('./utilsBundleImpl').yaml;
export const ws: typeof import('../bundles/utils/node_modules/@types/ws') = require('./utilsBundleImpl').ws; export const ws: typeof import('../bundles/utils/node_modules/@types/ws') = require('./utilsBundleImpl').ws;