From 377404448ce1f070a216ac937792b29251fd57aa Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Mon, 20 Jul 2020 10:35:42 -0700 Subject: [PATCH] devops: add script to generate shared object => package mapping (#3022) We use this mapping to provide recommendations on which packages to install on Linux distributions. References #2745 --- utils/linux-browser-dependencies/.gitignore | 2 + utils/linux-browser-dependencies/README.md | 28 +++++ .../inside_docker/list_dependencies.js | 102 ++++++++++++++++++ .../inside_docker/process.sh | 19 ++++ utils/linux-browser-dependencies/run.sh | 35 ++++++ 5 files changed, 186 insertions(+) create mode 100644 utils/linux-browser-dependencies/.gitignore create mode 100644 utils/linux-browser-dependencies/README.md create mode 100644 utils/linux-browser-dependencies/inside_docker/list_dependencies.js create mode 100755 utils/linux-browser-dependencies/inside_docker/process.sh create mode 100755 utils/linux-browser-dependencies/run.sh diff --git a/utils/linux-browser-dependencies/.gitignore b/utils/linux-browser-dependencies/.gitignore new file mode 100644 index 0000000000..54f19fb231 --- /dev/null +++ b/utils/linux-browser-dependencies/.gitignore @@ -0,0 +1,2 @@ +RUN_RESULT +playwright.tar.gz diff --git a/utils/linux-browser-dependencies/README.md b/utils/linux-browser-dependencies/README.md new file mode 100644 index 0000000000..f5d2c8ff61 --- /dev/null +++ b/utils/linux-browser-dependencies/README.md @@ -0,0 +1,28 @@ +# Mapping distribution libraries to package names + +Playwright requires a set of packages on Linux distribution for browsers to work. +Before launching browser on Linux, Playwright uses `ldd` to make sure browsers have all +dependencies met. + +If this is not the case, Playwright suggests users packages to install to +meet the dependencies. This tool helps to maintain a map between package names +and shared libraries it provides, per distribution. + +## Usage + +To generate a map of browser library to package name on Ubuntu:bionic: + +```sh +$ ./run.sh ubuntu:bionic +``` + +Results will be saved to the `RUN_RESULT`. + + +## How it works + +The script does the following: + +1. Launches docker with given linux distribution +2. Installs playwright browsers inside the distribution +3. For every dependency that Playwright browsers miss inside the distribution, uses `apt-file` to reverse-search package with the library. diff --git a/utils/linux-browser-dependencies/inside_docker/list_dependencies.js b/utils/linux-browser-dependencies/inside_docker/list_dependencies.js new file mode 100644 index 0000000000..0c1016735e --- /dev/null +++ b/utils/linux-browser-dependencies/inside_docker/list_dependencies.js @@ -0,0 +1,102 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const util = require('util'); +const path = require('path'); +const {spawn} = require('child_process'); +const browserPaths = require('playwright/lib/install/browserPaths.js'); + +(async () => { + const allBrowsersPath = browserPaths.browsersPath(); + const {stdout} = await runCommand('find', [allBrowsersPath, '-executable', '-type', 'f']); + // lddPaths - files we want to run LDD against. + const lddPaths = stdout.trim().split('\n').map(f => f.trim()).filter(filePath => !filePath.toLowerCase().endsWith('.sh')); + // List of all shared libraries missing. + const missingDeps = new Set(); + // Multimap: reverse-mapping from shared library to requiring file. + const depsToLddPaths = new Map(); + await Promise.all(lddPaths.map(async lddPath => { + const deps = await missingFileDependencies(lddPath); + for (const dep of deps) { + missingDeps.add(dep); + let depsToLdd = depsToLddPaths.get(dep); + if (!depsToLdd) { + depsToLdd = new Set(); + depsToLddPaths.set(dep, depsToLdd); + } + depsToLdd.add(lddPath); + } + })); + console.log(`==== MISSING DEPENDENCIES: ${missingDeps.size} ====`); + console.log([...missingDeps].sort().join('\n')); + + console.log('{'); + for (const dep of missingDeps) { + const packages = await findPackages(dep); + if (packages.length === 0) { + console.log(` // UNRESOLVED: ${dep} `); + const depsToLdd = depsToLddPaths.get(dep); + for (const filePath of depsToLdd) + console.log(` // - required by ${filePath}`); + } else if (packages.length === 1) { + console.log(` "${dep}": "${packages[0]}",`); + } else { + console.log(` "${dep}": ${JSON.stringify(packages)},`); + } + } + console.log('}'); +})(); + +async function findPackages(libraryName) { + const {stdout} = await runCommand('apt-file', ['search', libraryName]); + if (!stdout.trim()) + return []; + const libs = stdout.trim().split('\n').map(line => line.split(':')[0]); + return [...new Set(libs)]; +} + +async function fileDependencies(filePath) { + const {stdout} = await lddAsync(filePath); + const deps = stdout.split('\n').map(line => { + line = line.trim(); + const missing = line.includes('not found'); + const name = line.split('=>')[0].trim(); + return {name, missing}; + }); + return deps; +} + +async function missingFileDependencies(filePath) { + const deps = await fileDependencies(filePath); + return deps.filter(dep => dep.missing).map(dep => dep.name); +} + +async function lddAsync(filePath) { + let LD_LIBRARY_PATH = []; + // Some shared objects inside browser sub-folders link against libraries that + // ship with the browser. We consider these to be included, so we want to account + // for them in the LD_LIBRARY_PATH. + for (let dirname = path.dirname(filePath); dirname !== '/'; dirname = path.dirname(dirname)) + LD_LIBRARY_PATH.push(dirname); + return await runCommand('ldd', [filePath], { + cwd: path.dirname(filePath), + env: { + ...process.env, + LD_LIBRARY_PATH: LD_LIBRARY_PATH.join(':'), + }, + }); +} + +function runCommand(command, args, options = {}) { + const childProcess = spawn(command, args, options); + + return new Promise((resolve) => { + let stdout = ''; + let stderr = ''; + childProcess.stdout.on('data', data => stdout += data); + childProcess.stderr.on('data', data => stderr += data); + childProcess.on('close', (code) => { + resolve({stdout, stderr, code}); + }); + }); +} diff --git a/utils/linux-browser-dependencies/inside_docker/process.sh b/utils/linux-browser-dependencies/inside_docker/process.sh new file mode 100755 index 0000000000..de27d01f42 --- /dev/null +++ b/utils/linux-browser-dependencies/inside_docker/process.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e +set +x + +# Install Node.js + +apt-get update && apt-get install -y curl && \ + curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ + apt-get install -y nodejs + +# Install apt-file +apt-get update && apt-get install -y apt-file && apt-file update + +# Install tip-of-tree playwright +mkdir /root/tmp && cd /root/tmp && npm init -y && npm i /root/hostfolder/playwright.tar.gz + +cp /root/hostfolder/inside_docker/list_dependencies.js /root/tmp/list_dependencies.js + +node list_dependencies.js | tee /root/hostfolder/RUN_RESULT diff --git a/utils/linux-browser-dependencies/run.sh b/utils/linux-browser-dependencies/run.sh new file mode 100755 index 0000000000..e3fa26ea26 --- /dev/null +++ b/utils/linux-browser-dependencies/run.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -e +set +x + +if [[ ($1 == '--help') || ($1 == '-h') ]]; then + echo "usage: $(basename $0) " + echo + echo "List mapping between browser dependencies to package names and save results in RUN_RESULT file." + echo "Example:" + echo "" + echo " $(basename $0) ubuntu:bionic" + echo "" + echo "NOTE: this requires Playwright dependencies to be installed with 'npm install'" + echo " and Playwright itself being built with 'npm run build'" + echo "" + exit 0 +fi + +if [[ $# == 0 ]]; then + echo "ERROR: please provide base image name, e.g. 'ubuntu:bionic'" + exit 1 +fi + +function cleanup() { + rm -f "playwright.tar.gz" +} + +trap "cleanup; cd $(pwd -P)" EXIT +cd "$(dirname "$0")" + +# We rely on `./playwright.tar.gz` to download browsers into the docker image. +node ../../packages/build_package.js playwright ./playwright.tar.gz + +docker run -v $PWD:/root/hostfolder --rm -it "$1" /root/hostfolder/inside_docker/process.sh +