test: speed up installation tests by not downloading from cdn multiple times (#27220)

tests/installation/expect.d.ts

@@ -20,7 +20,6 @@ declare global {
     namespace PlaywrightTest {
        interface Matchers<R, T> {
           toHaveLoggedSoftwareDownload(browsers: ("chromium" | "firefox" | "webkit" | "ffmpeg")[]): R;
-          toExistOnFS(): R;
        }
     }
 }

tests/installation/globalSetup.ts

@@ -13,13 +13,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import http from 'http';
+import https from 'https';
+import url from 'url';
+import type net from 'net';
+import debugLogger from 'debug';
 import path from 'path';
+import fs from 'fs';
 import { spawnAsync } from '../../packages/playwright-core/lib/utils/spawnAsync';
 import { rimraf } from 'playwright-core/lib/utilsBundle';
-import fs from 'fs';
 import { TMP_WORKSPACES } from './npmTest';
+import { createHttpServer } from '../../packages/playwright-core/lib/utils/network';
+import { calculateSha1 } from '../../packages/playwright-core/lib/utils/crypto';
 
 const PACKAGE_BUILDER_SCRIPT = path.join(__dirname, '..', '..', 'utils', 'pack_package.js');
+const BROWSERS_CACHE_DIR = path.join(TMP_WORKSPACES, 'npm-test-browsers-cache');
+const debug = debugLogger('itest');
 
 async function globalSetup() {
   await rimraf(TMP_WORKSPACES);
@@ -57,6 +67,61 @@ async function globalSetup() {
 
     await fs.promises.writeFile(path.join(__dirname, '.registry.json'), JSON.stringify(Object.fromEntries(builds)));
   }
+
+  const cdnProxyServer = createHttpServer(async (request: http.IncomingMessage, response: http.ServerResponse) => {
+    const requestedPath = url.parse(request.url!).path;
+    const cachedPath = path.join(BROWSERS_CACHE_DIR, calculateSha1(requestedPath));
+    const cachedPathMetaInfo = cachedPath + '.metainfo';
+
+    if (!fs.existsSync(cachedPath)) {
+      const realUrl = 'https://playwright.azureedge.net' + requestedPath;
+      debug(`[cdn proxy] downloading ${realUrl} headers=${JSON.stringify(request.headers)}`);
+      const headers = { ...request.headers };
+      delete headers['host'];
+      const options = {
+        ...url.parse(realUrl),
+        method: request.method,
+        headers,
+      };
+      const factory = options.protocol === 'https:' ? https : http;
+      let doneCallback = () => {};
+      const donePromise = new Promise<void>(f => doneCallback = () => {
+        debug(`[cdn proxy] downloading ${realUrl} finished`);
+        f();
+      });
+      const realRequest = factory.request(options, (realResponse: http.IncomingMessage) => {
+        const metaInfo = {
+          statusCode: realResponse.statusCode,
+          statusMessage: realResponse.statusMessage || '',
+          headers: realResponse.headers || {},
+        };
+        debug(`[cdn proxy] downloading ${realUrl} statusCode=${realResponse.statusCode}`);
+        fs.mkdirSync(path.dirname(cachedPathMetaInfo), { recursive: true });
+        fs.writeFileSync(cachedPathMetaInfo, JSON.stringify(metaInfo));
+        realResponse.pipe(fs.createWriteStream(cachedPath, { highWaterMark: 1024 * 1024 })).on('close', doneCallback).on('finish', doneCallback).on('error', doneCallback);
+      });
+      request.pipe(realRequest);
+      await donePromise;
+    }
+
+    const metaInfo = JSON.parse(fs.readFileSync(cachedPathMetaInfo, 'utf-8'));
+    response.writeHead(metaInfo.statusCode, metaInfo.statusMessage, metaInfo.headers);
+    const done = () => {
+      debug(`[cdn proxy] serving ${request.url!} finished`);
+      response.end();
+    };
+    fs.createReadStream(cachedPath, { highWaterMark: 1024 * 1024 }).pipe(response).on('close', done).on('error', done);
+    debug(`[cdn proxy] serving ${request.url!} from cached ${cachedPath}`);
+  });
+
+  cdnProxyServer.listen(0);
+  await new Promise(f => cdnProxyServer.once('listening', f));
+  process.env.CDN_PROXY_HOST = `http://127.0.0.1:${(cdnProxyServer.address() as net.AddressInfo).port}`;
+  console.log('Stared CDN proxy at ' + process.env.CDN_PROXY_HOST);
+
+  return async () => {
+    await new Promise(f => cdnProxyServer.close(f));
+  };
 }
 
 export default globalSetup;

tests/installation/npmTest.ts

@@ -17,7 +17,7 @@
 // eslint-disable-next-line spaced-comment
 /// <reference path="./expect.d.ts" />
 
-import { test as _test, expect as _expect } from '@playwright/test';
+import { _baseTest as _test, expect as _expect } from '@playwright/test';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
@@ -28,31 +28,11 @@ import type { CommonFixtures, CommonWorkerFixtures } from '../config/commonFixtu
 import { commonFixtures } from '../config/commonFixtures';
 import { removeFolders } from '../../packages/playwright-core/lib/utils/fileUtils';
 
-
 export const TMP_WORKSPACES = path.join(os.platform() === 'darwin' ? '/tmp' : os.tmpdir(), 'pwt', 'workspaces');
 
 const debug = debugLogger('itest');
 
-/**
- * A minimal NPM Registry Server that can serve local packages, or proxy to the upstream registry.
- * This is useful in test installation behavior of packages that aren't yet published. It's particularly helpful
- * when your installation requires transitive dependencies that are also not yet published.
- *
- * See https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md for information on the offical APIs.
- */
-
 _expect.extend({
-  async toExistOnFS(received: any) {
-    if (typeof received !== 'string')
-      throw new Error(`Expected argument to be a string.`);
-    try {
-      await fs.promises.access(received);
-      return { pass: true };
-    } catch (e) {
-      return { pass: false, message: () => 'file does not exist' };
-    }
-  },
-
   toHaveLoggedSoftwareDownload(received: any, browsers: ('chromium' | 'firefox' | 'webkit' | 'ffmpeg')[]) {
     if (typeof received !== 'string')
       throw new Error(`Expected argument to be a string.`);
@@ -77,25 +57,29 @@ _expect.extend({
 
 const expect = _expect;
 
-export type ExecOptions = { cwd?: string, env?: Record<string, string>, message?: string, expectToExitWithError?: boolean };
+type ExecOptions = { cwd?: string, env?: Record<string, string>, message?: string, expectToExitWithError?: boolean };
-export type ArgsOrOptions = [] | [...string[]] | [...string[], ExecOptions] | [ExecOptions];
+type ArgsOrOptions = [] | [...string[]] | [...string[], ExecOptions] | [ExecOptions];
 
-export type NPMTestFixtures = {
+type NPMTestOptions = {
-  _auto: void,
+  useRealCDN: boolean;
-  _browsersPath: string
+};
-  tmpWorkspace: string,
+
-  nodeMajorVersion: number,
+type NPMTestFixtures = {
+  _auto: void;
+  _browsersPath: string;
+  tmpWorkspace: string;
   installedSoftwareOnDisk: (registryPath?: string) => Promise<string[]>;
-  writeConfig: (allowGlobal: boolean) => Promise<void>,
+  writeConfig: (allowGlobal: boolean) => Promise<void>;
-  writeFiles: (nameToContents: Record<string, string>) => Promise<void>,
+  writeFiles: (nameToContents: Record<string, string>) => Promise<void>;
-  exec: (cmd: string, ...argsAndOrOptions: ArgsOrOptions) => Promise<string>
+  exec: (cmd: string, ...argsAndOrOptions: ArgsOrOptions) => Promise<string>;
-  tsc: (...argsAndOrOptions: ArgsOrOptions) => Promise<string>,
+  tsc: (...argsAndOrOptions: ArgsOrOptions) => Promise<string>;
-  registry: Registry,
+  registry: Registry;
 };
 
 export const test = _test
     .extend<CommonFixtures, CommonWorkerFixtures>(commonFixtures)
-    .extend<NPMTestFixtures>({
+    .extend<NPMTestFixtures & NPMTestOptions>({
+      useRealCDN: [false, { option: true }],
       _browsersPath: async ({ tmpWorkspace }, use) => use(path.join(tmpWorkspace, 'browsers')),
       _auto: [async ({ tmpWorkspace, exec, _browsersPath, writeConfig }, use) => {
         await exec('npm init -y');
@@ -119,9 +103,6 @@ export const test = _test
       }, {
         auto: true,
       }],
-      nodeMajorVersion: async ({}, use) => {
-        await use(+process.versions.node.split('.')[0]);
-      },
       writeFiles: async ({ tmpWorkspace }, use) => {
         await use(async (nameToContents: Record<string, string>) => {
           for (const [name, contents] of Object.entries(nameToContents))
@@ -164,7 +145,7 @@ export const test = _test
       installedSoftwareOnDisk: async ({ _browsersPath }, use) => {
         await use(async (registryPath?: string) => fs.promises.readdir(registryPath || _browsersPath).catch(() => []).then(files => files.map(f => f.split('-')[0]).filter(f => !f.startsWith('.'))));
       },
-      exec: async ({ registry, tmpWorkspace, _browsersPath }, use, testInfo) => {
+      exec: async ({ useRealCDN, tmpWorkspace, _browsersPath }, use, testInfo) => {
         await use(async (cmd: string, ...argsAndOrOptions: [] | [...string[]] | [...string[], ExecOptions] | [ExecOptions]) => {
           let args: string[] = [];
           let options: ExecOptions = {};
@@ -184,6 +165,7 @@ export const test = _test
                 'DISPLAY': process.env.DISPLAY,
                 'XAUTHORITY': process.env.XAUTHORITY,
                 'PLAYWRIGHT_BROWSERS_PATH': _browsersPath,
+                ...(useRealCDN ? {} : { PLAYWRIGHT_DOWNLOAD_HOST: process.env.CDN_PROXY_HOST }),
                 ...options.env,
               }
             });

tests/installation/playwright-cdn-failover-should-work.spec.ts

@@ -32,9 +32,10 @@ const parsedDownloads = (rawLogs: string) => {
   return out;
 };
 
+test.use({ useRealCDN: true });
 
 for (const cdn of CDNS) {
-  test(`playwright cdn failover should work (${cdn})`, async ({ exec, nodeMajorVersion, installedSoftwareOnDisk }) => {
+  test(`playwright cdn failover should work (${cdn})`, async ({ exec, installedSoftwareOnDisk }) => {
     await exec('npm i --foreground-scripts playwright');
     const result = await exec('npx playwright install', { env: { PW_TEST_CDN_THAT_SHOULD_WORK: cdn, DEBUG: 'pw:install' } });
     expect(result).toHaveLoggedSoftwareDownload(['chromium', 'ffmpeg', 'firefox', 'webkit']);
@@ -43,7 +44,6 @@ for (const cdn of CDNS) {
     for (const software of ['chromium', 'ffmpeg', 'firefox', 'webkit'])
       expect(dls).toContainEqual({ status: 200, name: software, url: expect.stringContaining(cdn) });
     await exec('node sanity.js playwright chromium firefox webkit');
-    if (nodeMajorVersion >= 14)
+    await exec('node esm-playwright.mjs');
-      await exec('node esm-playwright.mjs');
   });
 }

tests/installation/playwright-cdn-should-race-with-timeout.spec.ts

@@ -17,6 +17,8 @@ import http from 'http';
 import type { AddressInfo } from 'net';
 import { test, expect } from './npmTest';
 
+test.use({ useRealCDN: true });
+
 test(`playwright cdn should race with a timeout`, async ({ exec }) => {
   const server = http.createServer(() => {});
   await new Promise<void>(resolve => server.listen(0, resolve));

tests/installation/playwright-should-work.spec.ts

@@ -15,7 +15,9 @@
  */
 import { test, expect } from './npmTest';
 
-test(`playwright should work`, async ({ exec, nodeMajorVersion, installedSoftwareOnDisk }) => {
+test.use({ useRealCDN: true });
+
+test(`playwright should work`, async ({ exec, installedSoftwareOnDisk }) => {
   const result1 = await exec('npm i --foreground-scripts playwright');
   expect(result1).toHaveLoggedSoftwareDownload([]);
   expect(await installedSoftwareOnDisk()).toEqual([]);
@@ -25,6 +27,5 @@ test(`playwright should work`, async ({ exec, nodeMajorVersion, installedSoftwar
   expect(await installedSoftwareOnDisk()).toEqual(['chromium', 'ffmpeg', 'firefox', 'webkit']);
 
   await exec('node sanity.js playwright chromium firefox webkit');
-  if (nodeMajorVersion >= 14)
+  await exec('node esm-playwright.mjs');
-    await exec('node esm-playwright.mjs');
 });

tests/installation/playwright-test-should-work.spec.ts

@@ -16,7 +16,7 @@
 import { test } from './npmTest';
 import path from 'path';
 
-test('npm: @playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+test('npm: @playwright/test should work', async ({ exec, tmpWorkspace }) => {
   await exec('npm i --foreground-scripts @playwright/test');
   await exec('npx playwright test -c .', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' });
 
@@ -24,33 +24,30 @@ test('npm: @playwright/test should work', async ({ exec, nodeMajorVersion, tmpWo
   await exec('npx playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
   await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
   await exec('node sanity.js @playwright/test chromium firefox webkit');
-  if (nodeMajorVersion >= 14)
+  await exec('node', 'esm-playwright-test.mjs');
-    await exec('node', 'esm-playwright-test.mjs');
 });
 
-test('npm: playwright + @playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+test('npm: playwright + @playwright/test should work', async ({ exec, tmpWorkspace }) => {
   await exec('npm i --foreground-scripts playwright');
   await exec('npm i --foreground-scripts @playwright/test');
   await exec('npx playwright install');
   await exec('npx playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
   await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
   await exec('node sanity.js @playwright/test chromium firefox webkit');
-  if (nodeMajorVersion >= 14)
+  await exec('node', 'esm-playwright-test.mjs');
-    await exec('node', 'esm-playwright-test.mjs');
 });
 
-test('npm: @playwright/test + playwright-core should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+test('npm: @playwright/test + playwright-core should work', async ({ exec, tmpWorkspace }) => {
   await exec('npm i --foreground-scripts @playwright/test');
   await exec('npm i --foreground-scripts playwright-core');
   await exec('npx playwright install');
   await exec('npx playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
   await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
   await exec('node sanity.js @playwright/test chromium firefox webkit');
-  if (nodeMajorVersion >= 14)
+  await exec('node', 'esm-playwright-test.mjs');
-    await exec('node', 'esm-playwright-test.mjs');
 });
 
-test('yarn: @playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+test('yarn: @playwright/test should work', async ({ exec, tmpWorkspace }) => {
   await exec('yarn add @playwright/test');
   await exec('yarn playwright test -c .', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' });
 
@@ -58,17 +55,15 @@ test('yarn: @playwright/test should work', async ({ exec, nodeMajorVersion, tmpW
   await exec('yarn playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
   await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
   await exec('node sanity.js @playwright/test chromium firefox webkit');
-  if (nodeMajorVersion >= 14)
+  await exec('node', 'esm-playwright-test.mjs');
-    await exec('node', 'esm-playwright-test.mjs');
 });
 
-test('pnpm: @playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+test('pnpm: @playwright/test should work', async ({ exec, tmpWorkspace }) => {
   await exec('pnpm add @playwright/test');
   await exec('pnpm exec playwright test -c .', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' });
   await exec('pnpm exec playwright install');
   await exec('pnpm exec playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
   await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
   await exec('node sanity.js @playwright/test chromium firefox webkit');
-  if (nodeMajorVersion >= 14)
+  await exec('node', 'esm-playwright-test.mjs');
-    await exec('node', 'esm-playwright-test.mjs');
 });

tests/installation/playwright-xyz-should-work.spec.ts

@@ -17,7 +17,7 @@
 import { test, expect } from './npmTest';
 
 for (const browser of ['chromium', 'firefox', 'webkit']) {
-  test(`playwright-${browser} should work`, async ({ exec, nodeMajorVersion, installedSoftwareOnDisk }) => {
+  test(`playwright-${browser} should work`, async ({ exec, installedSoftwareOnDisk }) => {
     const pkg = `playwright-${browser}`;
     const result = await exec('npm i --foreground-scripts', pkg);
     const browserName = pkg.split('-')[1];
@@ -28,8 +28,7 @@ for (const browser of ['chromium', 'firefox', 'webkit']) {
     expect(await installedSoftwareOnDisk()).toEqual(expectedSoftware);
     expect(result).not.toContain(`To avoid unexpected behavior, please install your dependencies first`);
     await exec('node sanity.js', pkg, browser);
-    if (nodeMajorVersion >= 14)
+    await exec('node', `esm-${pkg}.mjs`);
-      await exec('node', `esm-${pkg}.mjs`);
   });
 }
 

tests/installation/registry.ts

@@ -16,14 +16,22 @@
 import crypto from 'crypto';
 import fs from 'fs';
 import type { Server } from 'http';
-import http from 'http';
+import type http from 'http';
 import https from 'https';
 import path from 'path';
 import { spawnAsync } from './spawnAsync';
+import { createHttpServer } from '../../packages/playwright-core/lib/utils/network';
 
 const kPublicNpmRegistry = 'https://registry.npmjs.org';
 const kContentTypeAbbreviatedMetadata = 'application/vnd.npm.install-v1+json';
 
+/**
+ * A minimal NPM Registry Server that can serve local packages, or proxy to the upstream registry.
+ * This is useful in test installation behavior of packages that aren't yet published. It's particularly helpful
+ * when your installation requires transitive dependencies that are also not yet published.
+ *
+ * See https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md for information on the offical APIs.
+ */
 export class Registry {
   private _workDir: string;
   private _url: string;
@@ -50,7 +58,7 @@ export class Registry {
 
     await Promise.all(Object.entries(packages).map(([pkg, tar]) => this._addPackage(pkg, tar)));
 
-    this._server = http.createServer(async (req, res) => {
+    this._server = createHttpServer(async (req: http.IncomingMessage, res: http.ServerResponse) => {
       // 1. Only support GET requests
       if (req.method !== 'GET')
         return res.writeHead(405).end();