chore: add browser like UA to browser fetcher (#11006)

Drive-by: unify all Playwright user agents across the board.

Co-authored-by: Andrey Lushnikov <lushnikov@chromium.org>

packages/playwright-core/src/server/fetch.ts

@@ -25,7 +25,7 @@ import zlib from 'zlib';
 import { HTTPCredentials } from '../../types/types';
 import * as channels from '../protocol/channels';
 import { TimeoutSettings } from '../utils/timeoutSettings';
-import { assert, createGuid, getPlaywrightVersion, monotonicTime } from '../utils/utils';
+import { assert, createGuid, getUserAgent, monotonicTime } from '../utils/utils';
 import { BrowserContext } from './browserContext';
 import { CookieStore, domainMatches } from './cookieStore';
 import { MultipartFormData } from './formData';
@@ -457,7 +457,7 @@ export class GlobalAPIRequestContext extends APIRequestContext {
     }
     this._options = {
       baseURL: options.baseURL,
-      userAgent: options.userAgent || `Playwright/${getPlaywrightVersion()}`,
+      userAgent: options.userAgent || getUserAgent(),
       extraHTTPHeaders: options.extraHTTPHeaders,
       ignoreHTTPSErrors: !!options.ignoreHTTPSErrors,
       httpCredentials: options.httpCredentials,

packages/playwright-core/src/utils/browserFetcher.ts

@@ -19,7 +19,7 @@ import extract from 'extract-zip';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
-import { existsAsync, download } from './utils';
+import { existsAsync, download, getUserAgent } from './utils';
 import { debugLogger } from './debugLogger';
 
 export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise<boolean> {
@@ -35,7 +35,8 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec
   try {
     await download(url, zipPath, {
       progressBarName,
-      log: debugLogger.log.bind(debugLogger, 'install')
+      log: debugLogger.log.bind(debugLogger, 'install'),
+      userAgent: getUserAgent(),
     });
     debugLogger.log('install', `extracting archive`);
     debugLogger.log('install', `-- zip: ${zipPath}`);

packages/playwright-core/src/utils/ubuntuVersion.ts

@@ -60,7 +60,7 @@ function getUbuntuVersionSyncInternal(): string {
   }
 }
 
-function parseUbuntuVersion(osReleaseText: string): string {
+export function parseOSReleaseText(osReleaseText: string): Map<string, string> {
   const fields = new Map();
   for (const line of osReleaseText.split('\n')) {
     const tokens = line.split('=');
@@ -72,11 +72,16 @@ function parseUbuntuVersion(osReleaseText: string): string {
       continue;
     fields.set(name.toLowerCase(), value);
   }
+  return fields;
+}
+
+function parseUbuntuVersion(osReleaseText: string): string {
+  const fields = parseOSReleaseText(osReleaseText);
   // For Linux mint
-  if (fields.get('distrib_id') && fields.get('distrib_id').toLowerCase() === 'ubuntu')
+  if (fields.get('distrib_id') && fields.get('distrib_id')?.toLowerCase() === 'ubuntu')
     return fields.get('distrib_release') || '';
 
-  if (!fields.get('name') || fields.get('name').toLowerCase() !== 'ubuntu')
+  if (!fields.get('name') || fields.get('name')?.toLowerCase() !== 'ubuntu')
     return '';
   return fields.get('version_id') || '';
 }

packages/playwright-core/src/utils/utils.ts

@@ -22,10 +22,10 @@ import * as crypto from 'crypto';
 import os from 'os';
 import http from 'http';
 import https from 'https';
-import { spawn, SpawnOptions } from 'child_process';
+import { spawn, SpawnOptions, execSync } from 'child_process';
 import { getProxyForUrl } from 'proxy-from-env';
 import * as URL from 'url';
-import { getUbuntuVersionSync } from './ubuntuVersion';
+import { getUbuntuVersionSync, parseOSReleaseText } from './ubuntuVersion';
 import { NameValue } from '../protocol/channels';
 import ProgressBar from 'progress';
 
@@ -115,8 +115,13 @@ export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequ
 
 type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
 type DownloadFileLogger = (message: string) => void;
+type DownloadFileOptions = {
+  progressCallback?: OnProgressCallback,
+  log?: DownloadFileLogger,
+  userAgent?: string
+};
 
-function downloadFile(url: string, destinationPath: string, options: {progressCallback?: OnProgressCallback, log?: DownloadFileLogger} = {}): Promise<{error: any}> {
+function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{error: any}> {
   const {
     progressCallback,
     log = () => {},
@@ -130,7 +135,12 @@ function downloadFile(url: string, destinationPath: string, options: {progressCa
 
   const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; });
 
-  httpRequest({ url }, response => {
+  httpRequest({
+    url,
+    headers: options.userAgent ? {
+      'User-Agent': options.userAgent,
+    } : undefined,
+  }, response => {
     log(`-- response status code: ${response.statusCode}`);
     if (response.statusCode !== 200) {
       const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
@@ -156,16 +166,19 @@ function downloadFile(url: string, destinationPath: string, options: {progressCa
   }
 }
 
+type DownloadOptions = {
+  progressBarName?: string,
+  retryCount?: number
+  log?: DownloadFileLogger
+  userAgent?: string
+};
+
 export async function download(
   url: string,
   destination: string,
-  options: {
+  options: DownloadOptions = {}
-		progressBarName?: string,
-		retryCount?: number
-    log?: DownloadFileLogger
-	} = {}
 ) {
-  const { progressBarName = 'file', retryCount = 3, log = () => {} } = options;
+  const { progressBarName = 'file', retryCount = 3, log = () => {}, userAgent } = options;
   for (let attempt = 1; attempt <= retryCount; ++attempt) {
     log(
         `downloading ${progressBarName} - attempt #${attempt}`
@@ -173,6 +186,7 @@ export async function download(
     const { error } = await downloadFile(url, destination, {
       progressCallback: getDownloadProgress(progressBarName),
       log,
+      userAgent,
     });
     if (!error) {
       log(`SUCCESS downloading ${progressBarName}`);
@@ -421,8 +435,55 @@ export function canAccessFile(file: string) {
   }
 }
 
-export function getUserAgent() {
+let cachedUserAgent: string | undefined;
-  return `Playwright/${getPlaywrightVersion()} (${os.arch()}/${os.platform()}/${os.release()})`;
+export function getUserAgent(): string {
+  if (cachedUserAgent)
+    return cachedUserAgent;
+  try {
+    cachedUserAgent = determineUserAgent();
+  } catch (e) {
+    cachedUserAgent = 'Playwright/unknown';
+  }
+  return cachedUserAgent;
+}
+
+function determineUserAgent(): string {
+  let osIdentifier = 'unknown';
+  let osVersion = 'unknown';
+  if (process.platform === 'win32') {
+    const version = os.release().split('.');
+    osIdentifier = 'windows';
+    osVersion = `${version[0]}.${version[1]}`;
+  } else if (process.platform === 'darwin') {
+    const version = execSync('sw_vers -productVersion').toString().trim().split('.');
+    osIdentifier = 'macOS';
+    osVersion = `${version[0]}.${version[1]}`;
+  } else if (process.platform === 'linux') {
+    try {
+      // List of /etc/os-release values for different distributions could be
+      // found here: https://gist.github.com/aslushnikov/8ceddb8288e4cf9db3039c02e0f4fb75
+      const osReleaseText = fs.readFileSync('/etc/os-release', 'utf8');
+      const fields = parseOSReleaseText(osReleaseText);
+      osIdentifier = fields.get('id') || 'unknown';
+      osVersion = fields.get('version_id') || 'unknown';
+    } catch (e) {
+      // Linux distribution without /etc/os-release.
+      // Default to linux/unknown.
+      osIdentifier = 'linux';
+    }
+  }
+
+  let langName = 'unknown';
+  let langVersion = 'unknown';
+  if (!process.env.PW_CLI_TARGET_LANG) {
+    langName = 'node';
+    langVersion = process.version.substring(1).split('.').slice(0, 2).join('.');
+  } else if (['node', 'python', 'java', 'csharp'].includes(process.env.PW_CLI_TARGET_LANG)) {
+    langName = process.env.PW_CLI_TARGET_LANG;
+    langVersion = process.env.PW_CLI_TARGET_LANG_VERSION ?? 'unknown';
+  }
+
+  return `Playwright/${getPlaywrightVersion()} (${os.arch()}; ${osIdentifier} ${osVersion}) ${langName}/${langVersion}`;
 }
 
 export function getPlaywrightVersion(majorMinorOnly = false) {

tests/global-fetch.spec.ts

@@ -15,6 +15,7 @@
  */
 
 import http from 'http';
+import os from 'os';
 import * as util from 'util';
 import { getPlaywrightVersion } from 'playwright-core/lib/utils/utils';
 import { expect, playwrightTest as it } from './config/browserTest';
@@ -177,13 +178,23 @@ it('should resolve url relative to gobal baseURL option', async ({ playwright, s
   expect(response.url()).toBe(server.EMPTY_PAGE);
 });
 
-it('should set playwright as user-agent', async ({ playwright, server }) => {
+it('should set playwright as user-agent', async ({ playwright, server, isWindows, isLinux, isMac }) => {
   const request = await playwright.request.newContext();
   const [serverRequest] = await Promise.all([
     server.waitForRequest('/empty.html'),
     request.get(server.EMPTY_PAGE)
   ]);
-  expect(serverRequest.headers['user-agent']).toBe('Playwright/' + getPlaywrightVersion());
+  const userAgentMasked = serverRequest.headers['user-agent']
+      .replace(os.arch(), '<ARCH>')
+      .replace(getPlaywrightVersion(), 'X.X.X')
+      .replace(/\d+/g, 'X');
+
+  if (isWindows)
+    expect(userAgentMasked).toBe('Playwright/X.X.X (<ARCH>; windows X.X) node/X.X');
+  else if (isLinux)
+    expect(userAgentMasked).toBe('Playwright/X.X.X (<ARCH>; ubuntu X.X) node/X.X');
+  else if (isMac)
+    expect(userAgentMasked).toBe('Playwright/X.X.X (<ARCH>; macOS X.X) node/X.X');
 });
 
 it('should be able to construct with context options', async ({ playwright, browserType, server }) => {