From 0228ba49926f804e0b6ebd9a1acd0ef27dabb4a8 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 29 Apr 2020 17:19:21 -0700 Subject: [PATCH] feat(registry): implement download registry (#1979) --- install-from-github.js | 18 ++--- packages/playwright-chromium/install.js | 4 +- packages/playwright-firefox/install.js | 4 +- packages/playwright-webkit/install.js | 4 +- packages/playwright/install.js | 4 +- src/install/browserFetcher.ts | 26 ++------ src/install/browserPaths.ts | 29 ++++---- src/install/installer.ts | 89 +++++++++++++++++++++++++ src/server/browserType.ts | 8 ++- 9 files changed, 129 insertions(+), 57 deletions(-) create mode 100644 src/install/installer.ts diff --git a/install-from-github.js b/install-from-github.js index 603997ccae..77f5ed4bcf 100644 --- a/install-from-github.js +++ b/install-from-github.js @@ -41,9 +41,7 @@ const rmAsync = util.promisify(require('rimraf')); if (outdatedFiles.some(Boolean)) { console.log(`Rebuilding playwright...`); try { - execSync('npm run build', { - stdio: 'ignore' - }); + execSync('npm run build'); } catch (e) { } } @@ -63,21 +61,17 @@ async function listFiles(dirpath) { } async function downloadAllBrowsersAndGenerateProtocolTypes() { - const { downloadBrowserWithProgressBar } = require('./lib/install/browserFetcher'); + const { installBrowsersWithProgressBar } = require('./lib/install/installer'); const protocolGenerator = require('./utils/protocol-types-generator'); const browserPaths = require('./lib/install/browserPaths'); + const browsersPath = browserPaths.browsersPath(__dirname); const browsers = require('./browsers.json')['browsers']; + await installBrowsersWithProgressBar(__dirname); for (const browser of browsers) { - if (await downloadBrowserWithProgressBar(__dirname, browser)) - await protocolGenerator.generateProtocol(browser.name, browserPaths.executablePath(__dirname, browser)).catch(console.warn); + const browserPath = browserPaths.browserDirectory(browsersPath, browser); + await protocolGenerator.generateProtocol(browser.name, browserPaths.executablePath(browserPath, browser)).catch(console.warn); } - // Cleanup stale revisions. - const directories = new Set(await readdirAsync(browserPaths.browsersPath(__dirname))); - for (const browser of browsers) - directories.delete(browserPaths.browserDirectory(__dirname, browser)); - await Promise.all([...directories].map(directory => rmAsync(directory))); - try { console.log('Generating types...'); execSync('npm run generate-types'); diff --git a/packages/playwright-chromium/install.js b/packages/playwright-chromium/install.js index c0ce8cb983..b249271f2b 100644 --- a/packages/playwright-chromium/install.js +++ b/packages/playwright-chromium/install.js @@ -14,6 +14,6 @@ * limitations under the License. */ -const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher'); +const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer'); -downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']); +installBrowsersWithProgressBar(__dirname); diff --git a/packages/playwright-firefox/install.js b/packages/playwright-firefox/install.js index c0ce8cb983..b249271f2b 100644 --- a/packages/playwright-firefox/install.js +++ b/packages/playwright-firefox/install.js @@ -14,6 +14,6 @@ * limitations under the License. */ -const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher'); +const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer'); -downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']); +installBrowsersWithProgressBar(__dirname); diff --git a/packages/playwright-webkit/install.js b/packages/playwright-webkit/install.js index c0ce8cb983..b249271f2b 100644 --- a/packages/playwright-webkit/install.js +++ b/packages/playwright-webkit/install.js @@ -14,6 +14,6 @@ * limitations under the License. */ -const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher'); +const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer'); -downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']); +installBrowsersWithProgressBar(__dirname); diff --git a/packages/playwright/install.js b/packages/playwright/install.js index c0ce8cb983..b249271f2b 100644 --- a/packages/playwright/install.js +++ b/packages/playwright/install.js @@ -14,6 +14,6 @@ * limitations under the License. */ -const { downloadBrowsersWithProgressBar } = require('playwright-core/lib/install/browserFetcher'); +const { installBrowsersWithProgressBar } = require('playwright-core/lib/install/installer'); -downloadBrowsersWithProgressBar(__dirname, require('./browsers.json')['browsers']); +installBrowsersWithProgressBar(__dirname); diff --git a/src/install/browserFetcher.ts b/src/install/browserFetcher.ts index da9f36fcd1..89583feb9c 100644 --- a/src/install/browserFetcher.ts +++ b/src/install/browserFetcher.ts @@ -75,12 +75,6 @@ function getDownloadUrl(browserName: BrowserName, platform: BrowserPlatform): st } } -export type DownloadOptions = { - browser: BrowserDescriptor, - packagePath: string, - serverHost?: string, -}; - function revisionURL(browser: BrowserDescriptor, platform = browserPaths.hostPlatform): string { const serverHost = getFromENV('PLAYWRIGHT_DOWNLOAD_HOST') || DEFAULT_DOWNLOAD_HOSTS[browser.name]; const urlTemplate = getDownloadUrl(browser.name, platform); @@ -88,19 +82,9 @@ function revisionURL(browser: BrowserDescriptor, platform = browserPaths.hostPla return util.format(urlTemplate, serverHost, browser.revision); } -export async function downloadBrowsersWithProgressBar(packagePath: string, browsers: BrowserDescriptor[]) { - if (getFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) { - logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set'); - return false; - } - for (const browser of browsers) - await downloadBrowserWithProgressBar(packagePath, browser); -} - -export async function downloadBrowserWithProgressBar(packagePath: string, browser: BrowserDescriptor): Promise { +export async function downloadBrowserWithProgressBar(browserPath: string, browser: BrowserDescriptor): Promise { const progressBarName = `${browser.name} v${browser.revision}`; - const targetDir = browserPaths.browserDirectory(packagePath, browser); - if (await existsAsync(targetDir)) { + if (await existsAsync(browserPath)) { // Already downloaded. return false; } @@ -126,8 +110,8 @@ export async function downloadBrowserWithProgressBar(packagePath: string, browse const zipPath = path.join(os.tmpdir(), `playwright-download-${browser.name}-${browserPaths.hostPlatform}-${browser.revision}.zip`); try { await downloadFile(url, zipPath, progress); - await extract(zipPath, {dir: targetDir}); - await chmodAsync(browserPaths.executablePath(packagePath, browser), 0o755); + await extract(zipPath, { dir: browserPath}); + await chmodAsync(browserPaths.executablePath(browserPath, browser)!, 0o755); } catch (e) { process.exitCode = 1; throw e; @@ -135,7 +119,7 @@ export async function downloadBrowserWithProgressBar(packagePath: string, browse if (await existsAsync(zipPath)) await unlinkAsync(zipPath); } - logPolitely(`${progressBarName} downloaded to ${targetDir}`); + logPolitely(`${progressBarName} downloaded to ${browserPath}`); return true; } diff --git a/src/install/browserPaths.ts b/src/install/browserPaths.ts index 96598a0384..a7bf8b62f8 100644 --- a/src/install/browserPaths.ts +++ b/src/install/browserPaths.ts @@ -18,7 +18,7 @@ import { execSync } from 'child_process'; import * as os from 'os'; import * as path from 'path'; -import { assert, getFromENV } from '../helper'; +import { getFromENV } from '../helper'; export type BrowserName = 'chromium'|'webkit'|'firefox'; export type BrowserPlatform = 'win32'|'win64'|'mac10.13'|'mac10.14'|'mac10.15'|'linux'; @@ -40,9 +40,10 @@ export const hostPlatform = ((): BrowserPlatform => { return platform as BrowserPlatform; })(); -function getRelativeExecutablePath(browserName: BrowserName): string[] | undefined { - if (browserName === 'chromium') { - return new Map([ +export function executablePath(browserPath: string, browser: BrowserDescriptor): string | undefined { + let tokens: string[] | undefined; + if (browser.name === 'chromium') { + tokens = new Map([ ['linux', ['chrome-linux', 'chrome']], ['mac10.13', ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium']], ['mac10.14', ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium']], @@ -52,8 +53,8 @@ function getRelativeExecutablePath(browserName: BrowserName): string[] | undefin ]).get(hostPlatform); } - if (browserName === 'firefox') { - return new Map([ + if (browser.name === 'firefox') { + tokens = new Map([ ['linux', ['firefox', 'firefox']], ['mac10.13', ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox']], ['mac10.14', ['firefox', 'Nightly.app', 'Contents', 'MacOS', 'firefox']], @@ -63,8 +64,8 @@ function getRelativeExecutablePath(browserName: BrowserName): string[] | undefin ]).get(hostPlatform); } - if (browserName === 'webkit') { - return new Map([ + if (browser.name === 'webkit') { + tokens = new Map([ ['linux', ['pw_run.sh']], ['mac10.13', undefined], ['mac10.14', ['pw_run.sh']], @@ -73,6 +74,7 @@ function getRelativeExecutablePath(browserName: BrowserName): string[] | undefin ['win64', ['Playwright.exe']], ]).get(hostPlatform); } + return tokens ? path.join(browserPath, ...tokens) : undefined; } export function browsersPath(packagePath: string): string { @@ -80,12 +82,11 @@ export function browsersPath(packagePath: string): string { return result || path.join(packagePath, '.local-browsers'); } -export function browserDirectory(packagePath: string, browser: BrowserDescriptor): string { - return path.join(browsersPath(packagePath), `${browser.name}-${browser.revision}`); +export function browserDirectory(browsersPath: string, browser: BrowserDescriptor): string { + return path.join(browsersPath, `${browser.name}-${browser.revision}`); } -export function executablePath(packagePath: string, browser: BrowserDescriptor): string { - const relativePath = getRelativeExecutablePath(browser.name); - assert(relativePath, `Unsupported platform for ${browser.name}: ${hostPlatform}`); - return path.join(browserDirectory(packagePath, browser), ...relativePath); +export function isBrowserDirectory(browserPath: string): boolean { + const baseName = path.basename(browserPath); + return baseName.startsWith('chromium-') || baseName.startsWith('firefox-') || baseName.startsWith('webkit-'); } diff --git a/src/install/installer.ts b/src/install/installer.ts new file mode 100644 index 0000000000..633b64c109 --- /dev/null +++ b/src/install/installer.ts @@ -0,0 +1,89 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as crypto from 'crypto'; +import { getFromENV, logPolitely } from '../helper'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as util from 'util'; +import * as removeFolder from 'rimraf'; +import * as browserPaths from '../install/browserPaths'; +import * as browserFetcher from '../install/browserFetcher'; + +const fsMkdirAsync = util.promisify(fs.mkdir.bind(fs)); +const fsExistsAsync = (path: string) => new Promise(f => fs.exists(path, f)); +const fsReaddirAsync = util.promisify(fs.readdir.bind(fs)); +const fsReadFileAsync = util.promisify(fs.readFile.bind(fs)); +const fsUnlinkAsync = util.promisify(fs.unlink.bind(fs)); +const fsWriteFileAsync = util.promisify(fs.writeFile.bind(fs)); +const removeFolderAsync = util.promisify(removeFolder); + +export async function installBrowsersWithProgressBar(packagePath: string) { + const browsersPath = browserPaths.browsersPath(packagePath); + const linksDir = path.join(browsersPath, '.links'); + + if (getFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) { + logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set'); + return false; + } + if (!await fsExistsAsync(browsersPath)) + await fsMkdirAsync(browsersPath); + if (!await fsExistsAsync(linksDir)) + await fsMkdirAsync(linksDir); + + await fsWriteFileAsync(path.join(linksDir, sha1(packagePath)), packagePath); + await validateCache(browsersPath, linksDir); +} + +async function validateCache(browsersPath: string, linksDir: string) { + // 1. Collect unused downloads and package descriptors. + const allBrowsers: browserPaths.BrowserDescriptor[] = []; + for (const fileName of await fsReaddirAsync(linksDir)) { + const linkPath = path.join(linksDir, fileName); + try { + const linkTarget = (await fsReadFileAsync(linkPath)).toString(); + const browsers = JSON.parse((await fsReadFileAsync(path.join(linkTarget, 'browsers.json'))).toString())['browsers']; + allBrowsers.push(...browsers); + } catch (e) { + logPolitely('Failed to process descriptor at ' + fileName); + await fsUnlinkAsync(linkPath).catch(e => {}); + } + } + + // 2. Delete all unused browsers. + let downloadedBrowsers = (await fsReaddirAsync(browsersPath)).map(file => path.join(browsersPath, file)); + downloadedBrowsers = downloadedBrowsers.filter(file => browserPaths.isBrowserDirectory(file)); + const directories = new Set(downloadedBrowsers); + directories.delete(path.join(browsersPath, '.links')); + for (const browser of allBrowsers) + directories.delete(browserPaths.browserDirectory(browsersPath, browser)); + for (const directory of directories) { + logPolitely('Removing unused browser at ' + directory); + await removeFolderAsync(directory).catch(e => {}); + } + + // 3. Install missing browsers. + for (const browser of allBrowsers) { + const browserPath = browserPaths.browserDirectory(browsersPath, browser); + await browserFetcher.downloadBrowserWithProgressBar(browserPath, browser); + } +} + +function sha1(data: string): string { + const sum = crypto.createHash('sha1'); + sum.update(data); + return sum.digest('hex'); +} diff --git a/src/server/browserType.ts b/src/server/browserType.ts index dd88db91ea..5f897b5e7c 100644 --- a/src/server/browserType.ts +++ b/src/server/browserType.ts @@ -54,14 +54,18 @@ export interface BrowserType { export abstract class AbstractBrowserType implements BrowserType { private _name: string; - private _executablePath: string; + private _executablePath: string | undefined; constructor(packagePath: string, browser: browserPaths.BrowserDescriptor) { this._name = browser.name; - this._executablePath = browserPaths.executablePath(packagePath, browser); + const browsersPath = browserPaths.browsersPath(packagePath); + const browserPath = browserPaths.browserDirectory(browsersPath, browser); + this._executablePath = browserPaths.executablePath(browserPath, browser); } executablePath(): string { + if (!this._executablePath) + throw new Error('Browser is not supported on current platform'); return this._executablePath; }