From b60c006c63e24ecdf8f89d8248b02ba143f76499 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 24 Apr 2020 19:14:10 -0700 Subject: [PATCH] chore: simplify and restructure downloads (#1974) --- .gitignore | 2 - download-browser.js | 91 +++----- index.js | 13 +- install-from-github.js | 33 +-- package-lock.json | 11 +- package.json | 1 + packages/playwright-chromium/index.js | 9 +- packages/playwright-chromium/install.js | 7 +- packages/playwright-firefox/index.js | 9 +- packages/playwright-firefox/install.js | 7 +- packages/playwright-webkit/index.js | 9 +- packages/playwright-webkit/install.js | 7 +- packages/playwright/index.js | 12 +- packages/playwright/install.js | 11 +- src/server/browserFetcher.ts | 279 +++++++++++++++--------- utils/check_availability.js | 6 +- utils/protocol-types-generator/index.js | 2 +- 17 files changed, 258 insertions(+), 251 deletions(-) diff --git a/.gitignore b/.gitignore index 647c46a0dc..3d0d6a506d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,4 @@ yarn.lock /src/webkit/protocol.ts lib/ playwright-*.tgz -/web.js -/web.js.map /types/* diff --git a/download-browser.js b/download-browser.js index 468ca71bc3..0e98d1bde1 100644 --- a/download-browser.js +++ b/download-browser.js @@ -13,78 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const fs = require('fs'); + const path = require('path'); const browserFetcher = require('./lib/server/browserFetcher.js'); const packageJSON = require('./package.json'); -function localDownloadOptions(browserName) { - const revision = packageJSON.playwright[`${browserName}_revision`]; - const downloadPath = path.join(__dirname, '.local-browsers', `${browserName}-${revision}`); - return { - browser: browserName, - progressBarBrowserName: `${browserName} r${revision}`, - revision, - downloadPath, - executablePath: browserFetcher.executablePath({browser: browserName, downloadPath}), - }; -} - -function downloadOptionsFromENV(packagePath, browserName) { +function resolveBrowser(packagePath, browserName) { const browsersPath = getFromENV('PLAYWRIGHT_BROWSERS_PATH'); - const downloadPath = browsersPath ? - path.join(browsersPath, 'v' + packageJSON.version, browserName) : - path.join(packagePath, '.local-browsers', browserName); - return { - downloadPath, - skipBrowserDownload: getFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD'), - progressBarBrowserName: `${browserName} for playwright v${packageJSON.version}`, - revision: packageJSON.playwright[`${browserName}_revision`], - browser: browserName, - host: getFromENV('PLAYWRIGHT_DOWNLOAD_HOST'), - executablePath: browserFetcher.executablePath({browser: browserName, downloadPath}), - }; + const baseDir = browsersPath || path.join(packagePath, '.local-browsers'); + const browserRevision = packageJSON.playwright[`${browserName}_revision`]; + return { baseDir, browserRevision }; } -async function downloadBrowserWithProgressBar(options) { - if (options.skipBrowserDownload) { - logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set'); - return; - } - let progressBar = null; - let lastDownloadedBytes = 0; - function progress(downloadedBytes, totalBytes) { - if (!progressBar) { - const ProgressBar = require('progress'); - progressBar = new ProgressBar(`Downloading ${options.progressBarBrowserName} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, - }); - } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); - } - await browserFetcher.downloadBrowser({...options, progress}).catch(e => { - process.exitCode = 1; - throw e; +function executablePath(packagePath, browserName) { + const { baseDir, browserRevision } = resolveBrowser(packagePath, browserName); + return browserFetcher.executablePath(baseDir, browserName, browserRevision); +} + +function targetDirectory(packagePath, browserName) { + const { baseDir, browserRevision } = resolveBrowser(packagePath, browserName); + return browserFetcher.targetDirectory(baseDir, browserName, browserRevision); +} + +async function downloadBrowserWithProgressBar(packagePath, browserName) { + const { baseDir, browserRevision } = resolveBrowser(packagePath, browserName); + if (getFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) + return browserFetcher.downloadBrowserWithProgressBar(null); + return browserFetcher.downloadBrowserWithProgressBar({ + baseDir, + browserName, + browserRevision, + progressBarName: `${browserName} for playwright v${packageJSON.version}`, + serverHost: getFromENV('PLAYWRIGHT_DOWNLOAD_HOST'), }); - logPolitely(`${options.progressBarBrowserName} downloaded to ${options.downloadPath}`); -} - -function toMegabytes(bytes) { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; -} - -function logPolitely(toBeLogged) { - const logLevel = process.env.npm_config_loglevel; - const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel) > -1; - - if (!logLevelDisplay) - console.log(toBeLogged); } function getFromENV(name) { @@ -94,4 +55,4 @@ function getFromENV(name) { return value; } -module.exports = {downloadBrowserWithProgressBar, downloadOptionsFromENV, localDownloadOptions}; +module.exports = { targetDirectory, executablePath, downloadBrowserWithProgressBar }; diff --git a/index.js b/index.js index 48353433ce..05a5e7b874 100644 --- a/index.js +++ b/index.js @@ -15,18 +15,15 @@ */ const fs = require('fs'); const path = require('path'); -const {Playwright} = require('./lib/server/playwright.js'); -const {localDownloadOptions} = require('./download-browser.js'); +const { Playwright } = require('./lib/server/playwright.js'); +const { executablePath } = require('./download-browser.js'); const playwright = new Playwright({ browsers: ['webkit', 'chromium', 'firefox'], }); -if (fs.existsSync(path.join(__dirname, '.local-browsers'))) { - playwright.chromium._executablePath = localDownloadOptions('chromium').executablePath; - playwright.firefox._executablePath = localDownloadOptions('firefox').executablePath; - playwright.webkit._executablePath = localDownloadOptions('webkit').executablePath; -} +playwright.chromium._executablePath = executablePath(__dirname, 'chromium'); +playwright.firefox._executablePath = executablePath(__dirname, 'firefox'); +playwright.webkit._executablePath = executablePath(__dirname, 'webkit'); module.exports = playwright; - diff --git a/install-from-github.js b/install-from-github.js index 7e0ca77e0c..b84fa3e557 100644 --- a/install-from-github.js +++ b/install-from-github.js @@ -64,33 +64,20 @@ async function listFiles(dirpath) { } async function downloadAllBrowsersAndGenerateProtocolTypes() { - const {downloadBrowserWithProgressBar, localDownloadOptions} = require('./download-browser'); + const { targetDirectory, executablePath, downloadBrowserWithProgressBar } = require('./download-browser'); const protocolGenerator = require('./utils/protocol-types-generator'); - const chromiumOptions = localDownloadOptions('chromium'); - const firefoxOptions = localDownloadOptions('firefox'); - const webkitOptions = localDownloadOptions('webkit'); - if (!(await existsAsync(chromiumOptions.downloadPath))) { - await downloadBrowserWithProgressBar(chromiumOptions); - await protocolGenerator.generateChromiumProtocol(chromiumOptions.executablePath).catch(console.warn); - } - if (!(await existsAsync(firefoxOptions.downloadPath))) { - await downloadBrowserWithProgressBar(firefoxOptions); - await protocolGenerator.generateFirefoxProtocol(firefoxOptions.executablePath).catch(console.warn); - } - if (!(await existsAsync(webkitOptions.downloadPath))) { - await downloadBrowserWithProgressBar(webkitOptions); - await protocolGenerator.generateWebKitProtocol(webkitOptions.downloadPath).catch(console.warn); - } + if (await downloadBrowserWithProgressBar(__dirname, 'chromium')) + await protocolGenerator.generateChromiumProtocol(executablePath(__dirname, 'chromium')).catch(console.warn); + if (await downloadBrowserWithProgressBar(__dirname, 'firefox')) + await protocolGenerator.generateFirefoxProtocol(executablePath(__dirname, 'firefox')).catch(console.warn); + if (await downloadBrowserWithProgressBar(__dirname, 'webkit')) + await protocolGenerator.generateWebKitProtocol(executablePath(__dirname, 'webkit')).catch(console.warn); // Cleanup stale revisions. const directories = new Set(await readdirAsync(path.join(__dirname, '.local-browsers'))); - directories.delete(chromiumOptions.downloadPath); - directories.delete(firefoxOptions.downloadPath); - directories.delete(webkitOptions.downloadPath); - // cleanup old browser directories. - directories.add(path.join(__dirname, '.local-chromium')); - directories.add(path.join(__dirname, '.local-firefox')); - directories.add(path.join(__dirname, '.local-webkit')); + directories.delete(targetDirectory(__dirname, 'chromium')); + directories.delete(targetDirectory(__dirname, 'firefox')); + directories.delete(targetDirectory(__dirname, 'webkit')); await Promise.all([...directories].map(directory => rmAsync(directory))); try { diff --git a/package-lock.json b/package-lock.json index 05efd75c14..94495ab05b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "playwright-core", - "version": "0.14.0-post", + "version": "1.0.0-post", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -91,6 +91,15 @@ "@types/node": "*" } }, + "@types/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-bPOsfCZ4tsTlKiBjBhKnM8jpY5nmIll166IPD58D92hR7G7kZDfx5iB9wGF4NfZrdKolebjeAr3GouYkSGoJ/A==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/proxy-from-env": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/proxy-from-env/-/proxy-from-env-1.0.0.tgz", diff --git a/package.json b/package.json index 3f49f2f9c5..819f6b9401 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "@types/mime": "^2.0.1", "@types/node": "^10.17.17", "@types/pngjs": "^3.4.0", + "@types/progress": "^2.0.3", "@types/proxy-from-env": "^1.0.0", "@types/rimraf": "^2.0.2", "@types/ws": "^6.0.1", diff --git a/packages/playwright-chromium/index.js b/packages/playwright-chromium/index.js index 50776eb99c..63494f4c50 100644 --- a/packages/playwright-chromium/index.js +++ b/packages/playwright-chromium/index.js @@ -13,15 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const path = require('path'); -const {Playwright} = require('playwright-core/lib/server/playwright.js'); -const {downloadOptionsFromENV} = require('playwright-core/download-browser.js'); + +const { Playwright } = require('playwright-core/lib/server/playwright.js'); +const { executablePath } = require('playwright-core/download-browser.js'); const playwright = new Playwright({ browsers: ['chromium'], }); -playwright.chromium._executablePath = downloadOptionsFromENV(__dirname, 'chromium').executablePath; +playwright.chromium._executablePath = executablePath(__dirname, 'chromium'); module.exports = playwright; - diff --git a/packages/playwright-chromium/install.js b/packages/playwright-chromium/install.js index 9ebd44a236..eacdcb3199 100644 --- a/packages/playwright-chromium/install.js +++ b/packages/playwright-chromium/install.js @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const path = require('path'); -const fs = require('fs'); -const {downloadBrowserWithProgressBar, downloadOptionsFromENV} = require('playwright-core/download-browser'); + +const { downloadBrowserWithProgressBar } = require('playwright-core/download-browser'); (async function() { - await downloadBrowserWithProgressBar(downloadOptionsFromENV(__dirname, 'chromium')); + await downloadBrowserWithProgressBar(__dirname, 'chromium'); })(); diff --git a/packages/playwright-firefox/index.js b/packages/playwright-firefox/index.js index 663ebb0fda..0e0c97a380 100644 --- a/packages/playwright-firefox/index.js +++ b/packages/playwright-firefox/index.js @@ -13,15 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const path = require('path'); -const {Playwright} = require('playwright-core/lib/server/playwright.js'); -const {downloadOptionsFromENV} = require('playwright-core/download-browser.js'); + +const { Playwright } = require('playwright-core/lib/server/playwright.js'); +const { executablePath } = require('playwright-core/download-browser.js'); const playwright = new Playwright({ browsers: ['firefox'], }); -playwright.firefox._executablePath = downloadOptionsFromENV(__dirname, 'firefox').executablePath; +playwright.firefox._executablePath = executablePath(__dirname, 'firefox'); module.exports = playwright; - diff --git a/packages/playwright-firefox/install.js b/packages/playwright-firefox/install.js index c699852865..8d899ac10e 100644 --- a/packages/playwright-firefox/install.js +++ b/packages/playwright-firefox/install.js @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const path = require('path'); -const fs = require('fs'); -const {downloadBrowserWithProgressBar, downloadOptionsFromENV} = require('playwright-core/download-browser'); + +const { downloadBrowserWithProgressBar } = require('playwright-core/download-browser'); (async function() { - await downloadBrowserWithProgressBar(downloadOptionsFromENV(__dirname, 'firefox')); + await downloadBrowserWithProgressBar(__dirname, 'firefox'); })(); diff --git a/packages/playwright-webkit/index.js b/packages/playwright-webkit/index.js index 2f55cca7b5..4b71f572d4 100644 --- a/packages/playwright-webkit/index.js +++ b/packages/playwright-webkit/index.js @@ -13,15 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const path = require('path'); -const {Playwright} = require('playwright-core/lib/server/playwright.js'); -const {downloadOptionsFromENV} = require('playwright-core/download-browser.js'); + +const { Playwright } = require('playwright-core/lib/server/playwright.js'); +const { executablePath } = require('playwright-core/download-browser.js'); const playwright = new Playwright({ browsers: ['webkit'], }); -playwright.webkit._executablePath = downloadOptionsFromENV(__dirname, 'webkit').executablePath; +playwright.webkit._executablePath = executablePath(__dirname, 'webkit'); module.exports = playwright; - diff --git a/packages/playwright-webkit/install.js b/packages/playwright-webkit/install.js index 26ddc66c74..2fb0742778 100644 --- a/packages/playwright-webkit/install.js +++ b/packages/playwright-webkit/install.js @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const path = require('path'); -const fs = require('fs'); -const {downloadBrowserWithProgressBar, downloadOptionsFromENV} = require('playwright-core/download-browser'); + +const { downloadBrowserWithProgressBar } = require('playwright-core/download-browser'); (async function() { - await downloadBrowserWithProgressBar(downloadOptionsFromENV(__dirname, 'webkit')); + await downloadBrowserWithProgressBar(__dirname, 'webkit'); })(); diff --git a/packages/playwright/index.js b/packages/playwright/index.js index ef1fd91094..3ff74a36f1 100644 --- a/packages/playwright/index.js +++ b/packages/playwright/index.js @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const path = require('path'); -const {Playwright} = require('playwright-core/lib/server/playwright.js'); -const {downloadOptionsFromENV} = require('playwright-core/download-browser.js'); + +const { Playwright } = require('playwright-core/lib/server/playwright.js'); +const { executablePath } = require('playwright-core/download-browser.js'); const playwright = new Playwright({ browsers: ['webkit', 'chromium', 'firefox'], }); -playwright.chromium._executablePath = downloadOptionsFromENV(__dirname, 'chromium').executablePath; -playwright.webkit._executablePath = downloadOptionsFromENV(__dirname, 'webkit').executablePath; -playwright.firefox._executablePath = downloadOptionsFromENV(__dirname, 'firefox').executablePath; +playwright.chromium._executablePath = executablePath(__dirname, 'chromium'); +playwright.webkit._executablePath = executablePath(__dirname, 'webkit'); +playwright.firefox._executablePath = executablePath(__dirname, 'firefox'); module.exports = playwright; diff --git a/packages/playwright/install.js b/packages/playwright/install.js index 39fcab0ee7..7a16e96ef2 100644 --- a/packages/playwright/install.js +++ b/packages/playwright/install.js @@ -13,12 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const path = require('path'); -const fs = require('fs'); -const {downloadBrowserWithProgressBar, downloadOptionsFromENV} = require('playwright-core/download-browser'); + + const { downloadBrowserWithProgressBar } = require('playwright-core/download-browser'); (async function() { - await downloadBrowserWithProgressBar(downloadOptionsFromENV(__dirname, 'chromium')); - await downloadBrowserWithProgressBar(downloadOptionsFromENV(__dirname, 'firefox')); - await downloadBrowserWithProgressBar(downloadOptionsFromENV(__dirname, 'webkit')); + await downloadBrowserWithProgressBar(__dirname, 'chromium'); + await downloadBrowserWithProgressBar(__dirname, 'firefox'); + await downloadBrowserWithProgressBar(__dirname, 'webkit'); })(); diff --git a/src/server/browserFetcher.ts b/src/server/browserFetcher.ts index eb0f22004e..f50de20e2d 100644 --- a/src/server/browserFetcher.ts +++ b/src/server/browserFetcher.ts @@ -15,164 +15,225 @@ * limitations under the License. */ +import { execSync } from 'child_process'; import * as extract from 'extract-zip'; import * as fs from 'fs'; -import * as os from 'os'; -import * as util from 'util'; -import { execSync } from 'child_process'; import * as ProxyAgent from 'https-proxy-agent'; +import * as os from 'os'; import * as path from 'path'; +import * as ProgressBar from 'progress'; import { getProxyForUrl } from 'proxy-from-env'; import * as URL from 'url'; +import * as util from 'util'; import { assert } from '../helper'; const unlinkAsync = util.promisify(fs.unlink.bind(fs)); const chmodAsync = util.promisify(fs.chmod.bind(fs)); const existsAsync = (path: string): Promise => new Promise(resolve => fs.stat(path, err => resolve(!err))); +export type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; +export type BrowserName = ('chromium'|'webkit'|'firefox'); +export type BrowserPlatform = ('win32'|'win64'|'mac10.13'|'mac10.14'|'mac10.15'|'linux'); + const DEFAULT_DOWNLOAD_HOSTS = { chromium: 'https://storage.googleapis.com', firefox: 'https://playwright.azureedge.net', webkit: 'https://playwright.azureedge.net', }; -const DOWNLOAD_URLS = { - chromium: { - 'linux': '%s/chromium-browser-snapshots/Linux_x64/%d/chrome-linux.zip', - 'mac10.13': '%s/chromium-browser-snapshots/Mac/%d/chrome-mac.zip', - 'mac10.14': '%s/chromium-browser-snapshots/Mac/%d/chrome-mac.zip', - 'mac10.15': '%s/chromium-browser-snapshots/Mac/%d/chrome-mac.zip', - 'win32': '%s/chromium-browser-snapshots/Win/%d/chrome-win.zip', - 'win64': '%s/chromium-browser-snapshots/Win_x64/%d/chrome-win.zip', - }, - firefox: { - 'linux': '%s/builds/firefox/%s/firefox-linux.zip', - 'mac10.13': '%s/builds/firefox/%s/firefox-mac.zip', - 'mac10.14': '%s/builds/firefox/%s/firefox-mac.zip', - 'mac10.15': '%s/builds/firefox/%s/firefox-mac.zip', - 'win32': '%s/builds/firefox/%s/firefox-win32.zip', - 'win64': '%s/builds/firefox/%s/firefox-win64.zip', - }, - webkit: { - 'linux': '%s/builds/webkit/%s/minibrowser-gtk-wpe.zip', - 'mac10.13': undefined, - 'mac10.14': '%s/builds/webkit/%s/minibrowser-mac-10.14.zip', - 'mac10.15': '%s/builds/webkit/%s/minibrowser-mac-10.15.zip', - 'win32': '%s/builds/webkit/%s/minibrowser-win64.zip', - 'win64': '%s/builds/webkit/%s/minibrowser-win64.zip', - }, -}; - -const RELATIVE_EXECUTABLE_PATHS = { - chromium: { - 'linux': ['chrome-linux', 'chrome'], - 'mac10.13': ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'], - 'mac10.14': ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'], - 'mac10.15': ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'], - 'win32': ['chrome-win', 'chrome.exe'], - 'win64': ['chrome-win', 'chrome.exe'], - }, - firefox: { - 'linux': ['firefox', 'firefox'], - 'mac10.13': ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox'], - 'mac10.14': ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox'], - 'mac10.15': ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox'], - 'win32': ['firefox', 'firefox.exe'], - 'win64': ['firefox', 'firefox.exe'], - }, - webkit: { - 'linux': ['pw_run.sh'], - 'mac10.13': undefined, - 'mac10.14': ['pw_run.sh'], - 'mac10.15': ['pw_run.sh'], - 'win32': ['Playwright.exe'], - 'win64': ['Playwright.exe'], - }, -}; - -export type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; -export type BrowserName = ('chromium'|'webkit'|'firefox'); -export type BrowserPlatform = ('win32'|'win64'|'mac10.13'|'mac10.14'|'mac10.15'|'linux'); - -export type DownloadOptions = { - browser: BrowserName, - revision: string, - downloadPath: string, - platform?: BrowserPlatform, - host?: string, - progress?: OnProgressCallback, -}; - -const CURRENT_HOST_PLATFORM = ((): string => { +const hostPlatform = ((): BrowserPlatform => { const platform = os.platform(); if (platform === 'darwin') { const macVersion = execSync('sw_vers -productVersion').toString('utf8').trim().split('.').slice(0, 2).join('.'); - return `mac${macVersion}`; + return `mac${macVersion}` as BrowserPlatform; } if (platform === 'linux') return 'linux'; if (platform === 'win32') return os.arch() === 'x64' ? 'win64' : 'win32'; - return platform; + return platform as BrowserPlatform; })(); -function revisionURL(options: DownloadOptions): string { - const { - browser, - revision, - platform = CURRENT_HOST_PLATFORM, - host = DEFAULT_DOWNLOAD_HOSTS[browser], - } = options; - assert(revision, `'revision' must be specified`); - assert(DOWNLOAD_URLS[browser], 'Unsupported browser: ' + browser); - const urlTemplate = DOWNLOAD_URLS[browser][platform as BrowserPlatform]; - assert(urlTemplate, `ERROR: Playwright does not support ${browser} on ${platform}`); - return util.format(urlTemplate, host, revision); +function getDownloadUrl(browserName: BrowserName, platform?: BrowserPlatform): string | undefined { + platform = platform || hostPlatform; + if (browserName === 'chromium') { + return new Map([ + ['linux', '%s/chromium-browser-snapshots/Linux_x64/%d/chrome-linux.zip'], + ['mac10.13', '%s/chromium-browser-snapshots/Mac/%d/chrome-mac.zip'], + ['mac10.14', '%s/chromium-browser-snapshots/Mac/%d/chrome-mac.zip'], + ['mac10.15', '%s/chromium-browser-snapshots/Mac/%d/chrome-mac.zip'], + ['win32', '%s/chromium-browser-snapshots/Win/%d/chrome-win.zip'], + ['win64', '%s/chromium-browser-snapshots/Win_x64/%d/chrome-win.zip'], + ]).get(platform); + } + + if (browserName === 'firefox') { + return new Map([ + ['linux', '%s/builds/firefox/%s/firefox-linux.zip'], + ['mac10.13', '%s/builds/firefox/%s/firefox-mac.zip'], + ['mac10.14', '%s/builds/firefox/%s/firefox-mac.zip'], + ['mac10.15', '%s/builds/firefox/%s/firefox-mac.zip'], + ['win32', '%s/builds/firefox/%s/firefox-win32.zip'], + ['win64', '%s/builds/firefox/%s/firefox-win64.zip'], + ]).get(platform); + } + + if (browserName === 'webkit') { + return new Map([ + ['linux', '%s/builds/webkit/%s/minibrowser-gtk-wpe.zip'], + ['mac10.13', undefined], + ['mac10.14', '%s/builds/webkit/%s/minibrowser-mac-10.14.zip'], + ['mac10.15', '%s/builds/webkit/%s/minibrowser-mac-10.15.zip'], + ['win32', '%s/builds/webkit/%s/minibrowser-win64.zip'], + ['win64', '%s/builds/webkit/%s/minibrowser-win64.zip'], + ]).get(platform); + } } -export async function downloadBrowser(options: DownloadOptions): Promise { +function getRelativeExecutablePath(browserName: BrowserName): string[] | undefined { + if (browserName === 'chromium') { + return new Map([ + ['linux', ['chrome-linux', 'chrome']], + ['mac10.13', ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium']], + ['mac10.14', ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium']], + ['mac10.15', ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium']], + ['win32', ['chrome-win', 'chrome.exe']], + ['win64', ['chrome-win', 'chrome.exe']], + ]).get(hostPlatform); + } + + if (browserName === 'firefox') { + return new Map([ + ['linux', ['firefox', 'firefox']], + ['mac10.13', ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox']], + ['mac10.14', ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox']], + ['mac10.15', ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox']], + ['win32', ['firefox', 'firefox.exe']], + ['win64', ['firefox', 'firefox.exe']], + ]).get(hostPlatform); + } + + if (browserName === 'webkit') { + return new Map([ + ['linux', ['pw_run.sh']], + ['mac10.13', undefined], + ['mac10.14', ['pw_run.sh']], + ['mac10.15', ['pw_run.sh']], + ['win32', ['Playwright.exe']], + ['win64', ['Playwright.exe']], + ]).get(hostPlatform); + } +} + +export type DownloadOptions = { + baseDir: string, + browserName: BrowserName, + browserRevision: string, + progressBarName: string, + serverHost?: string, +}; + +function revisionURL(options: DownloadOptions, platform?: BrowserPlatform): string { const { - browser, - revision, - downloadPath, - platform = CURRENT_HOST_PLATFORM, - progress, + browserName, + browserRevision, + serverHost = DEFAULT_DOWNLOAD_HOSTS[browserName], } = options; - assert(downloadPath, '`downloadPath` must be provided'); - if (await existsAsync(downloadPath)) - return; + assert(browserRevision, `'revision' must be specified`); + const urlTemplate = getDownloadUrl(browserName, platform); + assert(urlTemplate, `ERROR: Playwright does not support ${browserName} on ${hostPlatform}`); + return util.format(urlTemplate, serverHost, browserRevision); +} + +export function targetDirectory(baseDir: string, browserName: string, browserRevision: string): string { + return path.join(baseDir, `${browserName}-${browserRevision}`); +} + +export function executablePath(baseDir: string, browserName: BrowserName, browserRevision: string): string { + const relativePath = getRelativeExecutablePath(browserName); + assert(relativePath, `Unsupported platform for ${browserName}: ${hostPlatform}`); + return path.join(targetDirectory(baseDir, browserName, browserRevision), ...relativePath); +} + +export async function downloadBrowserWithProgressBar(options: DownloadOptions | null): Promise { + if (!options) { + logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set'); + return false; + } + const { + baseDir, + browserName, + browserRevision, + progressBarName + } = options; + assert(baseDir, '`baseDir` must be provided'); + const targetDir = targetDirectory(baseDir, browserName, browserRevision); + if (await existsAsync(targetDir)) { + // Already downloaded. + return false; + } + + let progressBar: ProgressBar; + let lastDownloadedBytes = 0; + + function progress(downloadedBytes: number, totalBytes: number) { + if (!progressBar) { + progressBar = new ProgressBar(`Downloading ${progressBarName} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + }); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + } + const url = revisionURL(options); - const zipPath = path.join(os.tmpdir(), `playwright-download-${browser}-${platform}-${revision}.zip`); + const zipPath = path.join(os.tmpdir(), `playwright-download-${browserName}-${hostPlatform}-${browserRevision}.zip`); try { await downloadFile(url, zipPath, progress); - await extract(zipPath, {dir: downloadPath}); + await extract(zipPath, {dir: targetDir}); + await chmodAsync(executablePath(baseDir, browserName, browserRevision), 0o755); + } catch (e) { + process.exitCode = 1; + throw e; } finally { if (await existsAsync(zipPath)) await unlinkAsync(zipPath); } - await chmodAsync(executablePath(options), 0o755); + logPolitely(`${progressBarName} downloaded to ${targetDir}`); + return true; } -export function executablePath(options: DownloadOptions): string { - const { - browser, - downloadPath, - platform = CURRENT_HOST_PLATFORM, - } = options; - const relativePath = RELATIVE_EXECUTABLE_PATHS[browser][platform as BrowserPlatform]; - assert(relativePath, `Unsupported platform for ${browser}: ${platform}`); - return path.join(downloadPath, ...relativePath); +function toMegabytes(bytes: number) { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; } -export async function canDownload(options: DownloadOptions): Promise { - const url = revisionURL(options); +function logPolitely(toBeLogged: string) { + const logLevel = process.env.npm_config_loglevel; + const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel || '') > -1; + + if (!logLevelDisplay) + console.log(toBeLogged); // eslint-disable-line no-console +} + +export async function canDownload(browserName: BrowserName, browserRevision: string, platform: BrowserPlatform): Promise { + const url = revisionURL({ + baseDir: '', + browserName, + browserRevision, + progressBarName: '', + }, platform); let resolve: (result: boolean) => void = () => {}; const promise = new Promise(x => resolve = x); const request = httpRequest(url, 'HEAD', response => { resolve(response.statusCode === 200); }); request.on('error', (error: any) => { - console.error(error); + console.error(error); // eslint-disable-line no-console resolve(false); }); return promise; diff --git a/utils/check_availability.js b/utils/check_availability.js index 340d5527ca..685b05a4b6 100755 --- a/utils/check_availability.js +++ b/utils/check_availability.js @@ -20,7 +20,7 @@ const browserFetcher = require('../lib/server/browserFetcher.js'); const https = require('https'); const SUPPORTER_PLATFORMS = ['linux', 'mac', 'win32', 'win64']; -const fetcherOptions = SUPPORTER_PLATFORMS.map(platform => ({platform: platform === 'mac' ? 'mac10.15' : platform, browser: 'chromium'})); +const fetcherOptions = SUPPORTER_PLATFORMS.map(platform => platform === 'mac' ? 'mac10.15' : platform); const colors = { reset: '\x1b[0m', @@ -87,7 +87,7 @@ async function checkRangeAvailability(fromRevision, toRevision, stopWhenAllAvail table.drawRow([''].concat(SUPPORTER_PLATFORMS)); const inc = fromRevision < toRevision ? 1 : -1; for (let revision = fromRevision; revision !== toRevision; revision += inc) { - const allAvailable = await checkAndDrawRevisionAvailability(table, '', revision); + const allAvailable = await checkAndDrawRevisionAvailability(table, 'chromium', revision); if (allAvailable && stopWhenAllAvailable) break; } @@ -100,7 +100,7 @@ async function checkRangeAvailability(fromRevision, toRevision, stopWhenAllAvail * @return {boolean} */ async function checkAndDrawRevisionAvailability(table, name, revision) { - const promises = fetcherOptions.map(options => browserFetcher.canDownload({...options, revision})); + const promises = fetcherOptions.map(platform => browserFetcher.canDownload(name, revision, platform)); const availability = await Promise.all(promises); const allAvailable = availability.every(e => !!e); const values = [name + ' ' + (allAvailable ? colors.green + revision + colors.reset : revision)]; diff --git a/utils/protocol-types-generator/index.js b/utils/protocol-types-generator/index.js index 645207cc67..483979e245 100644 --- a/utils/protocol-types-generator/index.js +++ b/utils/protocol-types-generator/index.js @@ -26,7 +26,7 @@ async function generateChromiumProtocol(executablePath) { async function generateWebKitProtocol(folderPath) { const outputPath = path.join(__dirname, '..', '..', 'src', 'webkit', 'protocol.ts'); - const json = JSON.parse(await fs.promises.readFile(path.join(folderPath, 'protocol.json'), 'utf8')); + const json = JSON.parse(await fs.promises.readFile(path.join(folderPath, '..', 'protocol.json'), 'utf8')); await fs.promises.writeFile(outputPath, jsonToTS({domains: json})); console.log(`Wrote protocol.ts for WebKit to ${path.relative(process.cwd(), outputPath)}`); }