Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore: move install to Registry (#7433)

Browse source 
This is an effort to consolidate all handling of browser binaries in a single place.
This commit is contained in:
Dmitry Gozman 2021-07-06 20:59:16 -07:00 committed by GitHub
parent 50579ba27b
commit 47885db116
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 125 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
install-from-github.js
View file

@ -40,8 +40,8 @@ try {
}
console.log(`Downloading browsers...`);
const { installBrowsersWithProgressBar } = require('./lib/install/installer');
installBrowsersWithProgressBar().catch(e => {
const { installDefaultBrowsersForNpmInstall } = require('./lib/utils/registry');
installDefaultBrowsersForNpmInstall().catch(e => {
console.error(`Failed to install browsers, caused by\n${e.stack}`);
process.exit(1);
});

2
package.json
View file

@ -19,7 +19,7 @@
"test": "npm run basetest -- --config=tests/config/default.config.ts",
"eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext ts . || eslint --ext ts .",
"tsc": "tsc -p .",
"build-installer": "babel -s --extensions \".ts\" --out-dir lib/install/ src/install && babel -s --extensions \".ts\" --out-dir lib/utils/ src/utils",
"build-installer": "babel -s --extensions \".ts\" --out-dir lib/utils/ src/utils",
"doc": "node utils/doclint/cli.js",
"lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && node utils/generate_channels.js && node utils/generate_types/ --check-clean && npm run test-types",
"clean": "rimraf lib && rimraf src/generated/",

4
packages/common/install.js
View file

@ -14,6 +14,6 @@
* limitations under the License.
*/
const { installBrowsersWithProgressBar } = require('./lib/install/installer');
const { installDefaultBrowsersForNpmInstall } = require('./lib/utils/registry');
installBrowsersWithProgressBar();
installDefaultBrowsersForNpmInstall();

2
packages/installation-tests/sanity.js
View file

@ -33,7 +33,7 @@ const playwright = require(requireName);
// Requiring internals should work.
const errors = require(requireName + '/lib/utils/errors');
const installer = require(requireName + '/lib/install/installer');
const registry = require(requireName + '/lib/utils/registry');
(async () => {
for (const browserType of success) {

6
src/cli/cli.ts
View file

@ -22,7 +22,7 @@ import fs from 'fs';
import os from 'os';
import path from 'path';
import program from 'commander';
import { runDriver, runServer, printApiJson, launchBrowserServer, installBrowsers } from './driver';
import { runDriver, runServer, printApiJson, launchBrowserServer } from './driver';
import { showTraceViewer } from '../server/trace/viewer/traceViewer';
import * as playwright from '../..';
import { BrowserContext } from '../client/browserContext';
@ -126,7 +126,7 @@ program
try {
// Install default browsers when invoked without arguments.
if (!args.length) {
await installBrowsers();
await Registry.currentPackageRegistry().installBinaries();
return;
}
const browserNames: Set<BrowserName> = new Set(args.filter((browser: any) => allBrowserNames.has(browser)));
@ -139,7 +139,7 @@ program
if (browserNames.has('chromium') || browserChannels.has('chrome-beta') || browserChannels.has('chrome') || browserChannels.has('msedge') || browserChannels.has('msedge-beta'))
browserNames.add('ffmpeg');
if (browserNames.size)
await installBrowsers([...browserNames]);
await Registry.currentPackageRegistry().installBinaries([...browserNames]);
for (const browserChannel of browserChannels)
await installBrowserChannel(browserChannel);
} catch (e) {

6
src/cli/driver.ts
View file

@ -22,12 +22,10 @@ import { BrowserType } from '../client/browserType';
import { LaunchServerOptions } from '../client/types';
import { DispatcherConnection } from '../dispatchers/dispatcher';
import { PlaywrightDispatcher } from '../dispatchers/playwrightDispatcher';
import { installBrowsersWithProgressBar } from '../install/installer';
import { Transport } from '../protocol/transport';
import { PlaywrightServer } from '../remote/playwrightServer';
import { createPlaywright } from '../server/playwright';
import { gracefullyCloseAll } from '../server/processLauncher';
import { BrowserName } from '../utils/registry';
export function printApiJson() {
// Note: this file is generated by build-playwright-driver.sh
@ -66,7 +64,3 @@ export async function launchBrowserServer(browserName: string, configFile?: stri
const server = await browserType.launchServer(options);
console.log(server.wsEndpoint());
}
export async function installBrowsers(browserNames?: BrowserName[]) {
await installBrowsersWithProgressBar(browserNames);
}

114
src/install/installer.ts
View file

@ -1,114 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import path from 'path';
import lockfile from 'proper-lockfile';
import {Registry, allBrowserNames, isBrowserDirectory, BrowserName, registryDirectory} from '../utils/registry';
import * as browserFetcher from './browserFetcher';
import { getAsBooleanFromENV, calculateSha1, removeFolders } from '../utils/utils';
const fsExistsAsync = (filePath: string) => fs.promises.readFile(filePath).then(() => true).catch(e => false);
const PACKAGE_PATH = path.join(__dirname, '..', '..');
export async function installBrowsersWithProgressBar(browserNames: BrowserName[] = Registry.currentPackageRegistry().installByDefault()) {
// PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD should have a value of 0 or 1
if (getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) {
browserFetcher.logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set');
return false;
}
await fs.promises.mkdir(registryDirectory, { recursive: true });
const lockfilePath = path.join(registryDirectory, '__dirlock');
const releaseLock = await lockfile.lock(registryDirectory, {
retries: {
retries: 10,
// Retry 20 times during 10 minutes with
// exponential back-off.
// See documentation at: https://www.npmjs.com/package/retry#retrytimeoutsoptions
factor: 1.27579,
},
onCompromised: (err: Error) => {
throw new Error(`${err.message} Path: ${lockfilePath}`);
},
lockfilePath,
});
const linksDir = path.join(registryDirectory, '.links');
try {
await fs.promises.mkdir(linksDir, { recursive: true });
await fs.promises.writeFile(path.join(linksDir, calculateSha1(PACKAGE_PATH)), PACKAGE_PATH);
await validateCache(linksDir, browserNames);
} finally {
await releaseLock();
}
}
async function validateCache(linksDir: string, browserNames: BrowserName[]) {
// 1. Collect used downloads and package descriptors.
const usedBrowserPaths: Set<string> = new Set();
for (const fileName of await fs.promises.readdir(linksDir)) {
const linkPath = path.join(linksDir, fileName);
let linkTarget = '';
try {
linkTarget = (await fs.promises.readFile(linkPath)).toString();
const linkRegistry = new Registry(linkTarget);
for (const browserName of allBrowserNames) {
if (!linkRegistry.isSupportedBrowser(browserName))
continue;
const usedBrowserPath = linkRegistry.browserDirectory(browserName);
const browserRevision = linkRegistry.revision(browserName);
// Old browser installations don't have marker file.
const shouldHaveMarkerFile = (browserName === 'chromium' && browserRevision >= 786218) ||
(browserName === 'firefox' && browserRevision >= 1128) ||
(browserName === 'webkit' && browserRevision >= 1307) ||
// All new applications have a marker file right away.
(browserName !== 'firefox' && browserName !== 'chromium' && browserName !== 'webkit');
if (!shouldHaveMarkerFile || (await fsExistsAsync(markerFilePath(usedBrowserPath))))
usedBrowserPaths.add(usedBrowserPath);
}
} catch (e) {
await fs.promises.unlink(linkPath).catch(e => {});
}
}
// 2. Delete all unused browsers.
if (!getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_GC')) {
let downloadedBrowsers = (await fs.promises.readdir(registryDirectory)).map(file => path.join(registryDirectory, file));
downloadedBrowsers = downloadedBrowsers.filter(file => isBrowserDirectory(file));
const directories = new Set<string>(downloadedBrowsers);
for (const browserDirectory of usedBrowserPaths)
directories.delete(browserDirectory);
for (const directory of directories)
browserFetcher.logPolitely('Removing unused browser at ' + directory);
await removeFolders([...directories]);
}
// 3. Install missing browsers for this package.
const myRegistry = Registry.currentPackageRegistry();
for (const browserName of browserNames) {
await browserFetcher.downloadBrowserWithProgressBar(myRegistry, browserName).catch(e => {
throw new Error(`Failed to download ${browserName}, caused by\n${e.stack}`);
});
await fs.promises.writeFile(markerFilePath(myRegistry.browserDirectory(browserName)), '');
}
}
function markerFilePath(browserDirectory: string): string {
return path.join(browserDirectory, 'INSTALLATION_COMPLETE');
}

17
src/install/browserFetcher.ts → src/utils/browserFetcher.ts
View file

@ -20,16 +20,14 @@ import fs from 'fs';
import os from 'os';
import path from 'path';
import ProgressBar from 'progress';
import { BrowserName, Registry, hostPlatform } from '../utils/registry';
import { downloadFile, existsAsync } from '../utils/utils';
import { debugLogger } from '../utils/debugLogger';
import { downloadFile, existsAsync } from './utils';
import { debugLogger } from './debugLogger';
export async function downloadBrowserWithProgressBar(registry: Registry, browserName: BrowserName): Promise<boolean> {
const browserDirectory = registry.browserDirectory(browserName);
const progressBarName = `Playwright build of ${browserName} v${registry.revision(browserName)}`;
export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise<boolean> {
const progressBarName = `Playwright build of ${title}`;
if (await existsAsync(browserDirectory)) {
// Already downloaded.
debugLogger.log('install', `browser ${browserName} is already downloaded.`);
debugLogger.log('install', `browser ${title} is already downloaded.`);
return false;
}
@ -52,8 +50,8 @@ export async function downloadBrowserWithProgressBar(registry: Registry, browser
progressBar.tick(delta);
}
const url = registry.downloadURL(browserName);
const zipPath = path.join(os.tmpdir(), `playwright-download-${browserName}-${hostPlatform}-${registry.revision(browserName)}.zip`);
const url = downloadURL;
const zipPath = path.join(os.tmpdir(), `${downloadFileName}.zip`);
try {
for (let attempt = 1, N = 3; attempt <= N; ++attempt) {
debugLogger.log('install', `downloading ${progressBarName} - attempt #${attempt}`);
@ -77,7 +75,6 @@ export async function downloadBrowserWithProgressBar(registry: Registry, browser
debugLogger.log('install', `-- zip: ${zipPath}`);
debugLogger.log('install', `-- location: ${browserDirectory}`);
await extract(zipPath, { dir: browserDirectory});
const executablePath = registry.executablePath(browserName)!;
debugLogger.log('install', `fixing permissions at ${executablePath}`);
await fs.promises.chmod(executablePath, 0o755);
} catch (e) {

111
src/utils/registry.ts
View file

@ -18,9 +18,12 @@
import * as os from 'os';
import path from 'path';
import * as util from 'util';
import * as fs from 'fs';
import lockfile from 'proper-lockfile';
import { getUbuntuVersion, getUbuntuVersionSync } from './ubuntuVersion';
import { assert, getFromENV, getAsBooleanFromENV } from './utils';
import { assert, getFromENV, getAsBooleanFromENV, calculateSha1, removeFolders, existsAsync } from './utils';
import { installDependenciesLinux, installDependenciesWindows, validateDependenciesLinux, validateDependenciesWindows } from './dependencies';
import { downloadBrowserWithProgressBar, logPolitely } from './browserFetcher';
export type BrowserName = 'chromium'|'chromium-with-symbols'|'webkit'|'firefox'|'firefox-beta'|'ffmpeg';
export const allBrowserNames: Set<BrowserName> = new Set(['chromium', 'chromium-with-symbols', 'webkit', 'firefox', 'ffmpeg', 'firefox-beta']);
@ -288,7 +291,7 @@ export class Registry {
return path.join(registryDirectory, browser.browserDirectory);
}
revision(browserName: BrowserName): number {
private _revision(browserName: BrowserName): number {
const browser = this._descriptors.find(browser => browser.name === browserName);
assert(browser, `ERROR: Playwright does not support ${browserName}`);
return parseInt(browser.revision, 10);
@ -300,7 +303,7 @@ export class Registry {
return tokens ? path.join(browserDirectory, ...tokens) : undefined;
}
downloadURL(browserName: BrowserName): string {
private _downloadURL(browserName: BrowserName): string {
const browser = this._descriptors.find(browser => browser.name === browserName);
assert(browser, `ERROR: Playwright does not support ${browserName}`);
const envDownloadHost: { [key: string]: string } = {
@ -328,7 +331,7 @@ export class Registry {
return this._descriptors.some(browser => browser.name === browserName);
}
installByDefault(): BrowserName[] {
private _installByDefault(): BrowserName[] {
return this._descriptors.filter(browser => browser.installByDefault).map(browser => browser.name);
}
@ -378,7 +381,7 @@ export class Registry {
async installDeps(browserNames: BrowserName[]) {
const targets = new Set<'chromium' | 'firefox' | 'webkit' | 'tools'>();
if (!browserNames.length)
browserNames = this.installByDefault();
browserNames = this._installByDefault();
for (const browserName of browserNames) {
if (browserName === 'chromium' || browserName === 'chromium-with-symbols')
targets.add('chromium');
@ -393,4 +396,102 @@ export class Registry {
if (os.platform() === 'linux')
return await installDependenciesLinux(targets);
}
async installBinaries(browserNames?: BrowserName[]) {
if (!browserNames)
browserNames = this._installByDefault();
await fs.promises.mkdir(registryDirectory, { recursive: true });
const lockfilePath = path.join(registryDirectory, '__dirlock');
const releaseLock = await lockfile.lock(registryDirectory, {
retries: {
retries: 10,
// Retry 20 times during 10 minutes with
// exponential back-off.
// See documentation at: https://www.npmjs.com/package/retry#retrytimeoutsoptions
factor: 1.27579,
},
onCompromised: (err: Error) => {
throw new Error(`${err.message} Path: ${lockfilePath}`);
},
lockfilePath,
});
const linksDir = path.join(registryDirectory, '.links');
try {
// Create a link first, so that cache validation does not remove our own browsers.
await fs.promises.mkdir(linksDir, { recursive: true });
await fs.promises.writeFile(path.join(linksDir, calculateSha1(PACKAGE_PATH)), PACKAGE_PATH);
// Remove stale browsers.
await this._validateInstallationCache(linksDir);
// Install missing browsers for this package.
for (const browserName of browserNames) {
const revision = this._revision(browserName);
const browserDirectory = this.browserDirectory(browserName);
const title = `${browserName} v${revision}`;
const downloadFileName = `playwright-download-${browserName}-${hostPlatform}-${revision}`;
await downloadBrowserWithProgressBar(title, browserDirectory, this.executablePath(browserName)!, this._downloadURL(browserName), downloadFileName).catch(e => {
throw new Error(`Failed to download ${title}, caused by\n${e.stack}`);
});
await fs.promises.writeFile(markerFilePath(browserDirectory), '');
}
} finally {
await releaseLock();
}
}
private async _validateInstallationCache(linksDir: string) {
// 1. Collect used downloads and package descriptors.
const usedBrowserPaths: Set<string> = new Set();
for (const fileName of await fs.promises.readdir(linksDir)) {
const linkPath = path.join(linksDir, fileName);
let linkTarget = '';
try {
linkTarget = (await fs.promises.readFile(linkPath)).toString();
const linkRegistry = new Registry(linkTarget);
for (const browserName of allBrowserNames) {
if (!linkRegistry.isSupportedBrowser(browserName))
continue;
const usedBrowserPath = linkRegistry.browserDirectory(browserName);
const browserRevision = linkRegistry._revision(browserName);
// Old browser installations don't have marker file.
const shouldHaveMarkerFile = (browserName === 'chromium' && browserRevision >= 786218) ||
(browserName === 'firefox' && browserRevision >= 1128) ||
(browserName === 'webkit' && browserRevision >= 1307) ||
// All new applications have a marker file right away.
(browserName !== 'firefox' && browserName !== 'chromium' && browserName !== 'webkit');
if (!shouldHaveMarkerFile || (await existsAsync(markerFilePath(usedBrowserPath))))
usedBrowserPaths.add(usedBrowserPath);
}
} catch (e) {
await fs.promises.unlink(linkPath).catch(e => {});
}
}
// 2. Delete all unused browsers.
if (!getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_GC')) {
let downloadedBrowsers = (await fs.promises.readdir(registryDirectory)).map(file => path.join(registryDirectory, file));
downloadedBrowsers = downloadedBrowsers.filter(file => isBrowserDirectory(file));
const directories = new Set<string>(downloadedBrowsers);
for (const browserDirectory of usedBrowserPaths)
directories.delete(browserDirectory);
for (const directory of directories)
logPolitely('Removing unused browser at ' + directory);
await removeFolders([...directories]);
}
}
}
function markerFilePath(browserDirectory: string): string {
return path.join(browserDirectory, 'INSTALLATION_COMPLETE');
}
export async function installDefaultBrowsersForNpmInstall() {
// PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD should have a value of 0 or 1
if (getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) {
logPolitely('Skipping browsers download because `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD` env variable is set');
return false;
}
await Registry.currentPackageRegistry().installBinaries();
}

3
utils/check_deps.js
View file

@ -109,7 +109,6 @@ function listAllFiles(dir) {
const DEPS = {};
DEPS['src/protocol/'] = ['src/utils/'];
DEPS['src/install/'] = ['src/utils/'];
// Client depends on chromium protocol for types.
DEPS['src/client/'] = ['src/common/', 'src/utils/', 'src/protocol/', 'src/server/chromium/protocol.d.ts'];
@ -152,7 +151,7 @@ DEPS['src/web/traceViewer/ui/'] = ['src/common/', 'src/protocol/', 'src/web/trac
DEPS['src/remote/'] = ['src/client/', 'src/debug/', 'src/dispatchers/', 'src/server/', 'src/server/supplements/', 'src/server/electron/', 'src/server/trace/'];
// CLI should only use client-side features.
DEPS['src/cli/'] = ['src/cli/**', 'src/client/**', 'src/install/**', 'src/generated/', 'src/server/injected/', 'src/debug/injected/', 'src/server/trace/**', 'src/utils/**'];
DEPS['src/cli/'] = ['src/cli/**', 'src/client/**', 'src/generated/', 'src/server/injected/', 'src/debug/injected/', 'src/server/trace/**', 'src/utils/**'];
DEPS['src/server/supplements/recorder/recorderApp.ts'] = ['src/common/', 'src/utils/', 'src/server/', 'src/server/chromium/'];
DEPS['src/server/supplements/recorderSupplement.ts'] = ['src/server/snapshot/', ...DEPS['src/server/']];

4
utils/roll_browser.js
View file

@ -73,8 +73,8 @@ Example:
if (descriptor.installByDefault) {
// 3. Download new browser.
console.log('\nDownloading new browser...');
const { installBrowsersWithProgressBar } = require('../lib/install/installer');
await installBrowsersWithProgressBar();
const { installDefaultBrowsersForNpmInstall } = require('../lib/utils/registry');
await installDefaultBrowsersForNpmInstall();
// 4. Generate types.
console.log('\nGenerating protocol types...');