From fb22c859d6cb738ecf04682246fce142ec40ebc3 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Fri, 14 Jan 2022 11:46:17 +0100 Subject: [PATCH] chore: add browser like UA to browser fetcher (#11006) Drive-by: unify all Playwright user agents across the board. Co-authored-by: Andrey Lushnikov --- packages/playwright-core/src/server/fetch.ts | 4 +- .../src/utils/browserFetcher.ts | 5 +- .../src/utils/ubuntuVersion.ts | 11 ++- packages/playwright-core/src/utils/utils.ts | 85 ++++++++++++++++--- tests/global-fetch.spec.ts | 15 +++- 5 files changed, 99 insertions(+), 21 deletions(-) diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts index 52eb9d0660..664a106b12 100644 --- a/packages/playwright-core/src/server/fetch.ts +++ b/packages/playwright-core/src/server/fetch.ts @@ -25,7 +25,7 @@ import zlib from 'zlib'; import { HTTPCredentials } from '../../types/types'; import * as channels from '../protocol/channels'; import { TimeoutSettings } from '../utils/timeoutSettings'; -import { assert, createGuid, getPlaywrightVersion, monotonicTime } from '../utils/utils'; +import { assert, createGuid, getUserAgent, monotonicTime } from '../utils/utils'; import { BrowserContext } from './browserContext'; import { CookieStore, domainMatches } from './cookieStore'; import { MultipartFormData } from './formData'; @@ -457,7 +457,7 @@ export class GlobalAPIRequestContext extends APIRequestContext { } this._options = { baseURL: options.baseURL, - userAgent: options.userAgent || `Playwright/${getPlaywrightVersion()}`, + userAgent: options.userAgent || getUserAgent(), extraHTTPHeaders: options.extraHTTPHeaders, ignoreHTTPSErrors: !!options.ignoreHTTPSErrors, httpCredentials: options.httpCredentials, diff --git a/packages/playwright-core/src/utils/browserFetcher.ts b/packages/playwright-core/src/utils/browserFetcher.ts index 8836c1379e..5089a4debe 100644 --- a/packages/playwright-core/src/utils/browserFetcher.ts +++ b/packages/playwright-core/src/utils/browserFetcher.ts @@ -19,7 +19,7 @@ import extract from 'extract-zip'; import fs from 'fs'; import os from 'os'; import path from 'path'; -import { existsAsync, download } from './utils'; +import { existsAsync, download, getUserAgent } from './utils'; import { debugLogger } from './debugLogger'; export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise { @@ -35,7 +35,8 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec try { await download(url, zipPath, { progressBarName, - log: debugLogger.log.bind(debugLogger, 'install') + log: debugLogger.log.bind(debugLogger, 'install'), + userAgent: getUserAgent(), }); debugLogger.log('install', `extracting archive`); debugLogger.log('install', `-- zip: ${zipPath}`); diff --git a/packages/playwright-core/src/utils/ubuntuVersion.ts b/packages/playwright-core/src/utils/ubuntuVersion.ts index 74015b2195..ec4489cfcd 100644 --- a/packages/playwright-core/src/utils/ubuntuVersion.ts +++ b/packages/playwright-core/src/utils/ubuntuVersion.ts @@ -60,7 +60,7 @@ function getUbuntuVersionSyncInternal(): string { } } -function parseUbuntuVersion(osReleaseText: string): string { +export function parseOSReleaseText(osReleaseText: string): Map { const fields = new Map(); for (const line of osReleaseText.split('\n')) { const tokens = line.split('='); @@ -72,11 +72,16 @@ function parseUbuntuVersion(osReleaseText: string): string { continue; fields.set(name.toLowerCase(), value); } + return fields; +} + +function parseUbuntuVersion(osReleaseText: string): string { + const fields = parseOSReleaseText(osReleaseText); // For Linux mint - if (fields.get('distrib_id') && fields.get('distrib_id').toLowerCase() === 'ubuntu') + if (fields.get('distrib_id') && fields.get('distrib_id')?.toLowerCase() === 'ubuntu') return fields.get('distrib_release') || ''; - if (!fields.get('name') || fields.get('name').toLowerCase() !== 'ubuntu') + if (!fields.get('name') || fields.get('name')?.toLowerCase() !== 'ubuntu') return ''; return fields.get('version_id') || ''; } diff --git a/packages/playwright-core/src/utils/utils.ts b/packages/playwright-core/src/utils/utils.ts index 2fc74e2800..a17561ef6c 100644 --- a/packages/playwright-core/src/utils/utils.ts +++ b/packages/playwright-core/src/utils/utils.ts @@ -22,10 +22,10 @@ import * as crypto from 'crypto'; import os from 'os'; import http from 'http'; import https from 'https'; -import { spawn, SpawnOptions } from 'child_process'; +import { spawn, SpawnOptions, execSync } from 'child_process'; import { getProxyForUrl } from 'proxy-from-env'; import * as URL from 'url'; -import { getUbuntuVersionSync } from './ubuntuVersion'; +import { getUbuntuVersionSync, parseOSReleaseText } from './ubuntuVersion'; import { NameValue } from '../protocol/channels'; import ProgressBar from 'progress'; @@ -115,8 +115,13 @@ export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequ type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; type DownloadFileLogger = (message: string) => void; +type DownloadFileOptions = { + progressCallback?: OnProgressCallback, + log?: DownloadFileLogger, + userAgent?: string +}; -function downloadFile(url: string, destinationPath: string, options: {progressCallback?: OnProgressCallback, log?: DownloadFileLogger} = {}): Promise<{error: any}> { +function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{error: any}> { const { progressCallback, log = () => {}, @@ -130,7 +135,12 @@ function downloadFile(url: string, destinationPath: string, options: {progressCa const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; }); - httpRequest({ url }, response => { + httpRequest({ + url, + headers: options.userAgent ? { + 'User-Agent': options.userAgent, + } : undefined, + }, response => { log(`-- response status code: ${response.statusCode}`); if (response.statusCode !== 200) { const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`); @@ -156,16 +166,19 @@ function downloadFile(url: string, destinationPath: string, options: {progressCa } } +type DownloadOptions = { + progressBarName?: string, + retryCount?: number + log?: DownloadFileLogger + userAgent?: string +}; + export async function download( url: string, destination: string, - options: { - progressBarName?: string, - retryCount?: number - log?: DownloadFileLogger - } = {} + options: DownloadOptions = {} ) { - const { progressBarName = 'file', retryCount = 3, log = () => {} } = options; + const { progressBarName = 'file', retryCount = 3, log = () => {}, userAgent } = options; for (let attempt = 1; attempt <= retryCount; ++attempt) { log( `downloading ${progressBarName} - attempt #${attempt}` @@ -173,6 +186,7 @@ export async function download( const { error } = await downloadFile(url, destination, { progressCallback: getDownloadProgress(progressBarName), log, + userAgent, }); if (!error) { log(`SUCCESS downloading ${progressBarName}`); @@ -421,8 +435,55 @@ export function canAccessFile(file: string) { } } -export function getUserAgent() { - return `Playwright/${getPlaywrightVersion()} (${os.arch()}/${os.platform()}/${os.release()})`; +let cachedUserAgent: string | undefined; +export function getUserAgent(): string { + if (cachedUserAgent) + return cachedUserAgent; + try { + cachedUserAgent = determineUserAgent(); + } catch (e) { + cachedUserAgent = 'Playwright/unknown'; + } + return cachedUserAgent; +} + +function determineUserAgent(): string { + let osIdentifier = 'unknown'; + let osVersion = 'unknown'; + if (process.platform === 'win32') { + const version = os.release().split('.'); + osIdentifier = 'windows'; + osVersion = `${version[0]}.${version[1]}`; + } else if (process.platform === 'darwin') { + const version = execSync('sw_vers -productVersion').toString().trim().split('.'); + osIdentifier = 'macOS'; + osVersion = `${version[0]}.${version[1]}`; + } else if (process.platform === 'linux') { + try { + // List of /etc/os-release values for different distributions could be + // found here: https://gist.github.com/aslushnikov/8ceddb8288e4cf9db3039c02e0f4fb75 + const osReleaseText = fs.readFileSync('/etc/os-release', 'utf8'); + const fields = parseOSReleaseText(osReleaseText); + osIdentifier = fields.get('id') || 'unknown'; + osVersion = fields.get('version_id') || 'unknown'; + } catch (e) { + // Linux distribution without /etc/os-release. + // Default to linux/unknown. + osIdentifier = 'linux'; + } + } + + let langName = 'unknown'; + let langVersion = 'unknown'; + if (!process.env.PW_CLI_TARGET_LANG) { + langName = 'node'; + langVersion = process.version.substring(1).split('.').slice(0, 2).join('.'); + } else if (['node', 'python', 'java', 'csharp'].includes(process.env.PW_CLI_TARGET_LANG)) { + langName = process.env.PW_CLI_TARGET_LANG; + langVersion = process.env.PW_CLI_TARGET_LANG_VERSION ?? 'unknown'; + } + + return `Playwright/${getPlaywrightVersion()} (${os.arch()}; ${osIdentifier} ${osVersion}) ${langName}/${langVersion}`; } export function getPlaywrightVersion(majorMinorOnly = false) { diff --git a/tests/global-fetch.spec.ts b/tests/global-fetch.spec.ts index 90a099dac3..e15fb32697 100644 --- a/tests/global-fetch.spec.ts +++ b/tests/global-fetch.spec.ts @@ -15,6 +15,7 @@ */ import http from 'http'; +import os from 'os'; import * as util from 'util'; import { getPlaywrightVersion } from 'playwright-core/lib/utils/utils'; import { expect, playwrightTest as it } from './config/browserTest'; @@ -177,13 +178,23 @@ it('should resolve url relative to gobal baseURL option', async ({ playwright, s expect(response.url()).toBe(server.EMPTY_PAGE); }); -it('should set playwright as user-agent', async ({ playwright, server }) => { +it('should set playwright as user-agent', async ({ playwright, server, isWindows, isLinux, isMac }) => { const request = await playwright.request.newContext(); const [serverRequest] = await Promise.all([ server.waitForRequest('/empty.html'), request.get(server.EMPTY_PAGE) ]); - expect(serverRequest.headers['user-agent']).toBe('Playwright/' + getPlaywrightVersion()); + const userAgentMasked = serverRequest.headers['user-agent'] + .replace(os.arch(), '') + .replace(getPlaywrightVersion(), 'X.X.X') + .replace(/\d+/g, 'X'); + + if (isWindows) + expect(userAgentMasked).toBe('Playwright/X.X.X (; windows X.X) node/X.X'); + else if (isLinux) + expect(userAgentMasked).toBe('Playwright/X.X.X (; ubuntu X.X) node/X.X'); + else if (isMac) + expect(userAgentMasked).toBe('Playwright/X.X.X (; macOS X.X) node/X.X'); }); it('should be able to construct with context options', async ({ playwright, browserType, server }) => {