From 47885db11688b0ce9139e192aa712c2f7d32a60e Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Tue, 6 Jul 2021 20:59:16 -0700 Subject: [PATCH] chore: move install to Registry (#7433) This is an effort to consolidate all handling of browser binaries in a single place. --- install-from-github.js | 4 +- package.json | 2 +- packages/common/install.js | 4 +- packages/installation-tests/sanity.js | 2 +- src/cli/cli.ts | 6 +- src/cli/driver.ts | 6 -- src/install/installer.ts | 114 ----------------------- src/{install => utils}/browserFetcher.ts | 17 ++-- src/utils/registry.ts | 111 +++++++++++++++++++++- utils/check_deps.js | 3 +- utils/roll_browser.js | 4 +- 11 files changed, 125 insertions(+), 148 deletions(-) delete mode 100644 src/install/installer.ts rename src/{install => utils}/browserFetcher.ts (81%) diff --git a/install-from-github.js b/install-from-github.js index 971506e776..6698bef74b 100644 --- a/install-from-github.js +++ b/install-from-github.js @@ -40,8 +40,8 @@ try { } console.log(`Downloading browsers...`); -const { installBrowsersWithProgressBar } = require('./lib/install/installer'); -installBrowsersWithProgressBar().catch(e => { +const { installDefaultBrowsersForNpmInstall } = require('./lib/utils/registry'); +installDefaultBrowsersForNpmInstall().catch(e => { console.error(`Failed to install browsers, caused by\n${e.stack}`); process.exit(1); }); diff --git a/package.json b/package.json index f2a706b6e5..c62405aec4 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test": "npm run basetest -- --config=tests/config/default.config.ts", "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext ts . || eslint --ext ts .", "tsc": "tsc -p .", - "build-installer": "babel -s --extensions \".ts\" --out-dir lib/install/ src/install && babel -s --extensions \".ts\" --out-dir lib/utils/ src/utils", + "build-installer": "babel -s --extensions \".ts\" --out-dir lib/utils/ src/utils", "doc": "node utils/doclint/cli.js", "lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && node utils/generate_channels.js && node utils/generate_types/ --check-clean && npm run test-types", "clean": "rimraf lib && rimraf src/generated/", diff --git a/packages/common/install.js b/packages/common/install.js index 67f7ba312a..8fd9adb039 100644 --- a/packages/common/install.js +++ b/packages/common/install.js @@ -14,6 +14,6 @@ * limitations under the License. */ -const { installBrowsersWithProgressBar } = require('./lib/install/installer'); +const { installDefaultBrowsersForNpmInstall } = require('./lib/utils/registry'); -installBrowsersWithProgressBar(); +installDefaultBrowsersForNpmInstall(); diff --git a/packages/installation-tests/sanity.js b/packages/installation-tests/sanity.js index 6cc6386e8f..18860e41c3 100644 --- a/packages/installation-tests/sanity.js +++ b/packages/installation-tests/sanity.js @@ -33,7 +33,7 @@ const playwright = require(requireName); // Requiring internals should work. const errors = require(requireName + '/lib/utils/errors'); -const installer = require(requireName + '/lib/install/installer'); +const registry = require(requireName + '/lib/utils/registry'); (async () => { for (const browserType of success) { diff --git a/src/cli/cli.ts b/src/cli/cli.ts index 0dcae0d23b..2ea219f12b 100755 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -22,7 +22,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import program from 'commander'; -import { runDriver, runServer, printApiJson, launchBrowserServer, installBrowsers } from './driver'; +import { runDriver, runServer, printApiJson, launchBrowserServer } from './driver'; import { showTraceViewer } from '../server/trace/viewer/traceViewer'; import * as playwright from '../..'; import { BrowserContext } from '../client/browserContext'; @@ -126,7 +126,7 @@ program try { // Install default browsers when invoked without arguments. if (!args.length) { - await installBrowsers(); + await Registry.currentPackageRegistry().installBinaries(); return; } const browserNames: Set = new Set(args.filter((browser: any) => allBrowserNames.has(browser))); @@ -139,7 +139,7 @@ program if (browserNames.has('chromium') || browserChannels.has('chrome-beta') || browserChannels.has('chrome') || browserChannels.has('msedge') || browserChannels.has('msedge-beta')) browserNames.add('ffmpeg'); if (browserNames.size) - await installBrowsers([...browserNames]); + await Registry.currentPackageRegistry().installBinaries([...browserNames]); for (const browserChannel of browserChannels) await installBrowserChannel(browserChannel); } catch (e) { diff --git a/src/cli/driver.ts b/src/cli/driver.ts index 86c8c54fec..9edb855c53 100644 --- a/src/cli/driver.ts +++ b/src/cli/driver.ts @@ -22,12 +22,10 @@ import { BrowserType } from '../client/browserType'; import { LaunchServerOptions } from '../client/types'; import { DispatcherConnection } from '../dispatchers/dispatcher'; import { PlaywrightDispatcher } from '../dispatchers/playwrightDispatcher'; -import { installBrowsersWithProgressBar } from '../install/installer'; import { Transport } from '../protocol/transport'; import { PlaywrightServer } from '../remote/playwrightServer'; import { createPlaywright } from '../server/playwright'; import { gracefullyCloseAll } from '../server/processLauncher'; -import { BrowserName } from '../utils/registry'; export function printApiJson() { // Note: this file is generated by build-playwright-driver.sh @@ -66,7 +64,3 @@ export async function launchBrowserServer(browserName: string, configFile?: stri const server = await browserType.launchServer(options); console.log(server.wsEndpoint()); } - -export async function installBrowsers(browserNames?: BrowserName[]) { - await installBrowsersWithProgressBar(browserNames); -} diff --git a/src/install/installer.ts b/src/install/installer.ts deleted file mode 100644 index 23b4f6bba4..0000000000 --- a/src/install/installer.ts +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs'; -import path from 'path'; -import lockfile from 'proper-lockfile'; -import {Registry, allBrowserNames, isBrowserDirectory, BrowserName, registryDirectory} from '../utils/registry'; -import * as browserFetcher from './browserFetcher'; -import { getAsBooleanFromENV, calculateSha1, removeFolders } from '../utils/utils'; - -const fsExistsAsync = (filePath: string) => fs.promises.readFile(filePath).then(() => true).catch(e => false); - -const PACKAGE_PATH = path.join(__dirname, '..', '..'); - -export async function installBrowsersWithProgressBar(browserNames: BrowserName[] = Registry.currentPackageRegistry().installByDefault()) { - // PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD should have a value of 0 or 1 - if (getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) { - browserFetcher.logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set'); - return false; - } - - await fs.promises.mkdir(registryDirectory, { recursive: true }); - const lockfilePath = path.join(registryDirectory, '__dirlock'); - const releaseLock = await lockfile.lock(registryDirectory, { - retries: { - retries: 10, - // Retry 20 times during 10 minutes with - // exponential back-off. - // See documentation at: https://www.npmjs.com/package/retry#retrytimeoutsoptions - factor: 1.27579, - }, - onCompromised: (err: Error) => { - throw new Error(`${err.message} Path: ${lockfilePath}`); - }, - lockfilePath, - }); - const linksDir = path.join(registryDirectory, '.links'); - - try { - await fs.promises.mkdir(linksDir, { recursive: true }); - await fs.promises.writeFile(path.join(linksDir, calculateSha1(PACKAGE_PATH)), PACKAGE_PATH); - await validateCache(linksDir, browserNames); - } finally { - await releaseLock(); - } -} - -async function validateCache(linksDir: string, browserNames: BrowserName[]) { - // 1. Collect used downloads and package descriptors. - const usedBrowserPaths: Set = new Set(); - for (const fileName of await fs.promises.readdir(linksDir)) { - const linkPath = path.join(linksDir, fileName); - let linkTarget = ''; - try { - linkTarget = (await fs.promises.readFile(linkPath)).toString(); - const linkRegistry = new Registry(linkTarget); - for (const browserName of allBrowserNames) { - if (!linkRegistry.isSupportedBrowser(browserName)) - continue; - const usedBrowserPath = linkRegistry.browserDirectory(browserName); - const browserRevision = linkRegistry.revision(browserName); - // Old browser installations don't have marker file. - const shouldHaveMarkerFile = (browserName === 'chromium' && browserRevision >= 786218) || - (browserName === 'firefox' && browserRevision >= 1128) || - (browserName === 'webkit' && browserRevision >= 1307) || - // All new applications have a marker file right away. - (browserName !== 'firefox' && browserName !== 'chromium' && browserName !== 'webkit'); - if (!shouldHaveMarkerFile || (await fsExistsAsync(markerFilePath(usedBrowserPath)))) - usedBrowserPaths.add(usedBrowserPath); - } - } catch (e) { - await fs.promises.unlink(linkPath).catch(e => {}); - } - } - - // 2. Delete all unused browsers. - if (!getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_GC')) { - let downloadedBrowsers = (await fs.promises.readdir(registryDirectory)).map(file => path.join(registryDirectory, file)); - downloadedBrowsers = downloadedBrowsers.filter(file => isBrowserDirectory(file)); - const directories = new Set(downloadedBrowsers); - for (const browserDirectory of usedBrowserPaths) - directories.delete(browserDirectory); - for (const directory of directories) - browserFetcher.logPolitely('Removing unused browser at ' + directory); - await removeFolders([...directories]); - } - - // 3. Install missing browsers for this package. - const myRegistry = Registry.currentPackageRegistry(); - for (const browserName of browserNames) { - await browserFetcher.downloadBrowserWithProgressBar(myRegistry, browserName).catch(e => { - throw new Error(`Failed to download ${browserName}, caused by\n${e.stack}`); - }); - await fs.promises.writeFile(markerFilePath(myRegistry.browserDirectory(browserName)), ''); - } -} - -function markerFilePath(browserDirectory: string): string { - return path.join(browserDirectory, 'INSTALLATION_COMPLETE'); -} - diff --git a/src/install/browserFetcher.ts b/src/utils/browserFetcher.ts similarity index 81% rename from src/install/browserFetcher.ts rename to src/utils/browserFetcher.ts index 57aec190a9..8b2fc4b211 100644 --- a/src/install/browserFetcher.ts +++ b/src/utils/browserFetcher.ts @@ -20,16 +20,14 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import ProgressBar from 'progress'; -import { BrowserName, Registry, hostPlatform } from '../utils/registry'; -import { downloadFile, existsAsync } from '../utils/utils'; -import { debugLogger } from '../utils/debugLogger'; +import { downloadFile, existsAsync } from './utils'; +import { debugLogger } from './debugLogger'; -export async function downloadBrowserWithProgressBar(registry: Registry, browserName: BrowserName): Promise { - const browserDirectory = registry.browserDirectory(browserName); - const progressBarName = `Playwright build of ${browserName} v${registry.revision(browserName)}`; +export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise { + const progressBarName = `Playwright build of ${title}`; if (await existsAsync(browserDirectory)) { // Already downloaded. - debugLogger.log('install', `browser ${browserName} is already downloaded.`); + debugLogger.log('install', `browser ${title} is already downloaded.`); return false; } @@ -52,8 +50,8 @@ export async function downloadBrowserWithProgressBar(registry: Registry, browser progressBar.tick(delta); } - const url = registry.downloadURL(browserName); - const zipPath = path.join(os.tmpdir(), `playwright-download-${browserName}-${hostPlatform}-${registry.revision(browserName)}.zip`); + const url = downloadURL; + const zipPath = path.join(os.tmpdir(), `${downloadFileName}.zip`); try { for (let attempt = 1, N = 3; attempt <= N; ++attempt) { debugLogger.log('install', `downloading ${progressBarName} - attempt #${attempt}`); @@ -77,7 +75,6 @@ export async function downloadBrowserWithProgressBar(registry: Registry, browser debugLogger.log('install', `-- zip: ${zipPath}`); debugLogger.log('install', `-- location: ${browserDirectory}`); await extract(zipPath, { dir: browserDirectory}); - const executablePath = registry.executablePath(browserName)!; debugLogger.log('install', `fixing permissions at ${executablePath}`); await fs.promises.chmod(executablePath, 0o755); } catch (e) { diff --git a/src/utils/registry.ts b/src/utils/registry.ts index 7a8a6627ae..61bfb7f95a 100644 --- a/src/utils/registry.ts +++ b/src/utils/registry.ts @@ -18,9 +18,12 @@ import * as os from 'os'; import path from 'path'; import * as util from 'util'; +import * as fs from 'fs'; +import lockfile from 'proper-lockfile'; import { getUbuntuVersion, getUbuntuVersionSync } from './ubuntuVersion'; -import { assert, getFromENV, getAsBooleanFromENV } from './utils'; +import { assert, getFromENV, getAsBooleanFromENV, calculateSha1, removeFolders, existsAsync } from './utils'; import { installDependenciesLinux, installDependenciesWindows, validateDependenciesLinux, validateDependenciesWindows } from './dependencies'; +import { downloadBrowserWithProgressBar, logPolitely } from './browserFetcher'; export type BrowserName = 'chromium'|'chromium-with-symbols'|'webkit'|'firefox'|'firefox-beta'|'ffmpeg'; export const allBrowserNames: Set = new Set(['chromium', 'chromium-with-symbols', 'webkit', 'firefox', 'ffmpeg', 'firefox-beta']); @@ -288,7 +291,7 @@ export class Registry { return path.join(registryDirectory, browser.browserDirectory); } - revision(browserName: BrowserName): number { + private _revision(browserName: BrowserName): number { const browser = this._descriptors.find(browser => browser.name === browserName); assert(browser, `ERROR: Playwright does not support ${browserName}`); return parseInt(browser.revision, 10); @@ -300,7 +303,7 @@ export class Registry { return tokens ? path.join(browserDirectory, ...tokens) : undefined; } - downloadURL(browserName: BrowserName): string { + private _downloadURL(browserName: BrowserName): string { const browser = this._descriptors.find(browser => browser.name === browserName); assert(browser, `ERROR: Playwright does not support ${browserName}`); const envDownloadHost: { [key: string]: string } = { @@ -328,7 +331,7 @@ export class Registry { return this._descriptors.some(browser => browser.name === browserName); } - installByDefault(): BrowserName[] { + private _installByDefault(): BrowserName[] { return this._descriptors.filter(browser => browser.installByDefault).map(browser => browser.name); } @@ -378,7 +381,7 @@ export class Registry { async installDeps(browserNames: BrowserName[]) { const targets = new Set<'chromium' | 'firefox' | 'webkit' | 'tools'>(); if (!browserNames.length) - browserNames = this.installByDefault(); + browserNames = this._installByDefault(); for (const browserName of browserNames) { if (browserName === 'chromium' || browserName === 'chromium-with-symbols') targets.add('chromium'); @@ -393,4 +396,102 @@ export class Registry { if (os.platform() === 'linux') return await installDependenciesLinux(targets); } + + async installBinaries(browserNames?: BrowserName[]) { + if (!browserNames) + browserNames = this._installByDefault(); + await fs.promises.mkdir(registryDirectory, { recursive: true }); + const lockfilePath = path.join(registryDirectory, '__dirlock'); + const releaseLock = await lockfile.lock(registryDirectory, { + retries: { + retries: 10, + // Retry 20 times during 10 minutes with + // exponential back-off. + // See documentation at: https://www.npmjs.com/package/retry#retrytimeoutsoptions + factor: 1.27579, + }, + onCompromised: (err: Error) => { + throw new Error(`${err.message} Path: ${lockfilePath}`); + }, + lockfilePath, + }); + const linksDir = path.join(registryDirectory, '.links'); + + try { + // Create a link first, so that cache validation does not remove our own browsers. + await fs.promises.mkdir(linksDir, { recursive: true }); + await fs.promises.writeFile(path.join(linksDir, calculateSha1(PACKAGE_PATH)), PACKAGE_PATH); + + // Remove stale browsers. + await this._validateInstallationCache(linksDir); + + // Install missing browsers for this package. + for (const browserName of browserNames) { + const revision = this._revision(browserName); + const browserDirectory = this.browserDirectory(browserName); + const title = `${browserName} v${revision}`; + const downloadFileName = `playwright-download-${browserName}-${hostPlatform}-${revision}`; + await downloadBrowserWithProgressBar(title, browserDirectory, this.executablePath(browserName)!, this._downloadURL(browserName), downloadFileName).catch(e => { + throw new Error(`Failed to download ${title}, caused by\n${e.stack}`); + }); + await fs.promises.writeFile(markerFilePath(browserDirectory), ''); + } + } finally { + await releaseLock(); + } + } + + private async _validateInstallationCache(linksDir: string) { + // 1. Collect used downloads and package descriptors. + const usedBrowserPaths: Set = new Set(); + for (const fileName of await fs.promises.readdir(linksDir)) { + const linkPath = path.join(linksDir, fileName); + let linkTarget = ''; + try { + linkTarget = (await fs.promises.readFile(linkPath)).toString(); + const linkRegistry = new Registry(linkTarget); + for (const browserName of allBrowserNames) { + if (!linkRegistry.isSupportedBrowser(browserName)) + continue; + const usedBrowserPath = linkRegistry.browserDirectory(browserName); + const browserRevision = linkRegistry._revision(browserName); + // Old browser installations don't have marker file. + const shouldHaveMarkerFile = (browserName === 'chromium' && browserRevision >= 786218) || + (browserName === 'firefox' && browserRevision >= 1128) || + (browserName === 'webkit' && browserRevision >= 1307) || + // All new applications have a marker file right away. + (browserName !== 'firefox' && browserName !== 'chromium' && browserName !== 'webkit'); + if (!shouldHaveMarkerFile || (await existsAsync(markerFilePath(usedBrowserPath)))) + usedBrowserPaths.add(usedBrowserPath); + } + } catch (e) { + await fs.promises.unlink(linkPath).catch(e => {}); + } + } + + // 2. Delete all unused browsers. + if (!getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_GC')) { + let downloadedBrowsers = (await fs.promises.readdir(registryDirectory)).map(file => path.join(registryDirectory, file)); + downloadedBrowsers = downloadedBrowsers.filter(file => isBrowserDirectory(file)); + const directories = new Set(downloadedBrowsers); + for (const browserDirectory of usedBrowserPaths) + directories.delete(browserDirectory); + for (const directory of directories) + logPolitely('Removing unused browser at ' + directory); + await removeFolders([...directories]); + } + } +} + +function markerFilePath(browserDirectory: string): string { + return path.join(browserDirectory, 'INSTALLATION_COMPLETE'); +} + +export async function installDefaultBrowsersForNpmInstall() { + // PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD should have a value of 0 or 1 + if (getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) { + logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set'); + return false; + } + await Registry.currentPackageRegistry().installBinaries(); } diff --git a/utils/check_deps.js b/utils/check_deps.js index 84b951b8d5..bfb6146778 100644 --- a/utils/check_deps.js +++ b/utils/check_deps.js @@ -109,7 +109,6 @@ function listAllFiles(dir) { const DEPS = {}; DEPS['src/protocol/'] = ['src/utils/']; -DEPS['src/install/'] = ['src/utils/']; // Client depends on chromium protocol for types. DEPS['src/client/'] = ['src/common/', 'src/utils/', 'src/protocol/', 'src/server/chromium/protocol.d.ts']; @@ -152,7 +151,7 @@ DEPS['src/web/traceViewer/ui/'] = ['src/common/', 'src/protocol/', 'src/web/trac DEPS['src/remote/'] = ['src/client/', 'src/debug/', 'src/dispatchers/', 'src/server/', 'src/server/supplements/', 'src/server/electron/', 'src/server/trace/']; // CLI should only use client-side features. -DEPS['src/cli/'] = ['src/cli/**', 'src/client/**', 'src/install/**', 'src/generated/', 'src/server/injected/', 'src/debug/injected/', 'src/server/trace/**', 'src/utils/**']; +DEPS['src/cli/'] = ['src/cli/**', 'src/client/**', 'src/generated/', 'src/server/injected/', 'src/debug/injected/', 'src/server/trace/**', 'src/utils/**']; DEPS['src/server/supplements/recorder/recorderApp.ts'] = ['src/common/', 'src/utils/', 'src/server/', 'src/server/chromium/']; DEPS['src/server/supplements/recorderSupplement.ts'] = ['src/server/snapshot/', ...DEPS['src/server/']]; diff --git a/utils/roll_browser.js b/utils/roll_browser.js index 7f7a236dfa..021caa3c47 100755 --- a/utils/roll_browser.js +++ b/utils/roll_browser.js @@ -73,8 +73,8 @@ Example: if (descriptor.installByDefault) { // 3. Download new browser. console.log('\nDownloading new browser...'); - const { installBrowsersWithProgressBar } = require('../lib/install/installer'); - await installBrowsersWithProgressBar(); + const { installDefaultBrowsersForNpmInstall } = require('../lib/utils/registry'); + await installDefaultBrowsersForNpmInstall(); // 4. Generate types. console.log('\nGenerating protocol types...');