test: add installation tests for yarn and pnpm (#26937)

.github/workflows/tests_primary.yml

@@ -235,6 +235,8 @@ jobs:
         DEBUG: pw:install
     - run: npm run build
     - run: npx playwright install-deps
+    - run: npm install -g yarn@1
+    - run: npm install -g pnpm@8
     - run: npm run itest
       if: matrix.os != 'ubuntu-latest'
       env:

tests/installation/npmTest.ts

@@ -86,6 +86,7 @@ export type NPMTestFixtures = {
   tmpWorkspace: string,
   nodeMajorVersion: number,
   installedSoftwareOnDisk: (registryPath?: string) => Promise<string[]>;
+  writeConfig: (allowGlobal: boolean) => Promise<void>,
   writeFiles: (nameToContents: Record<string, string>) => Promise<void>,
   exec: (cmd: string, ...argsAndOrOptions: ArgsOrOptions) => Promise<string>
   tsc: (...argsAndOrOptions: ArgsOrOptions) => Promise<string>,
@@ -96,11 +97,20 @@ export const test = _test
     .extend<CommonFixtures, CommonWorkerFixtures>(commonFixtures)
     .extend<NPMTestFixtures>({
       _browsersPath: async ({ tmpWorkspace }, use) => use(path.join(tmpWorkspace, 'browsers')),
-      _auto: [async ({ tmpWorkspace, exec, _browsersPath }, use) => {
+      _auto: [async ({ tmpWorkspace, exec, _browsersPath, writeConfig }, use) => {
         await exec('npm init -y');
         const sourceDir = path.join(__dirname, 'fixture-scripts');
         const contents = await fs.promises.readdir(sourceDir);
         await Promise.all(contents.map(f => fs.promises.copyFile(path.join(sourceDir, f), path.join(tmpWorkspace, f))));
+
+        const packages = JSON.parse((await fs.promises.readFile(path.join(__dirname, '.registry.json'), 'utf8')));
+        const prefixed = Object.fromEntries(Object.entries(packages).map(entry => ([entry[0], 'file:' + entry[1]])));
+        const packageJSON = JSON.parse(await fs.promises.readFile(path.join(tmpWorkspace, 'package.json'), 'utf-8'));
+        packageJSON.pnpm = { overrides: prefixed };
+        await fs.promises.writeFile(path.join(tmpWorkspace, 'package.json'), JSON.stringify(packageJSON, null, 2));
+
+        await writeConfig(false);
+
         await use();
         if (test.info().status === test.info().expectedStatus) {
           // Browsers are large, we remove them after each test to save disk space.
@@ -133,6 +143,24 @@ export const test = _test
         await use(registry);
         await registry.shutdown();
       },
+      writeConfig: async ({ tmpWorkspace, registry }, use, testInfo) => {
+        await use(async (allowGlobal: boolean) => {
+          const yarnLines = [
+            `registry "${registry.url()}/"`,
+            `cache "${testInfo.outputPath('npm_cache')}"`,
+          ];
+          const npmLines = [
+            `registry = ${registry.url()}/`,
+            `cache = ${testInfo.outputPath('npm_cache')}`,
+          ];
+          if (!allowGlobal) {
+            yarnLines.push(`prefix "${testInfo.outputPath('npm_global')}"`);
+            npmLines.push(`prefix = ${testInfo.outputPath('npm_global')}`);
+          }
+          await fs.promises.writeFile(path.join(tmpWorkspace, '.yarnrc'), yarnLines.join('\n'), 'utf-8');
+          await fs.promises.writeFile(path.join(tmpWorkspace, '.npmrc'), npmLines.join('\n'), 'utf-8');
+        });
+      },
       installedSoftwareOnDisk: async ({ _browsersPath }, use) => {
         await use(async (registryPath?: string) => fs.promises.readdir(registryPath || _browsersPath).catch(() => []).then(files => files.map(f => f.split('-')[0]).filter(f => !f.startsWith('.'))));
       },
@@ -156,9 +184,6 @@ export const test = _test
                 'DISPLAY': process.env.DISPLAY,
                 'XAUTHORITY': process.env.XAUTHORITY,
                 'PLAYWRIGHT_BROWSERS_PATH': _browsersPath,
-                'npm_config_cache': testInfo.outputPath('npm_cache'),
-                'npm_config_registry': registry.url(),
-                'npm_config_prefix': testInfo.outputPath('npm_global'),
                 ...options.env,
               }
             });

tests/installation/npx-global-help.spec.ts

@@ -15,8 +15,9 @@
  */
 import { test, expect } from './npmTest';
 
-test('npx playwright --help should not download browsers', async ({ exec, installedSoftwareOnDisk }) => {
+test('npx playwright --help should not download browsers', async ({ writeConfig, exec, installedSoftwareOnDisk }) => {
-  const result = await exec('npx playwright --help', { env: { npm_config_prefix: '' } }); // global npx and npm_config_prefix do not work together nicely (https://github.com/npm/cli/issues/5268)
+  await writeConfig(true);
+  const result = await exec('npx playwright --help');
   expect(result).toHaveLoggedSoftwareDownload([]);
   expect(await installedSoftwareOnDisk()).toEqual([]);
   expect(result).not.toContain(`To avoid unexpected behavior, please install your dependencies first`);

tests/installation/npx-global-install.spec.ts

@@ -15,10 +15,11 @@
  */
 import { test, expect } from './npmTest';
 
-test('npx playwright install global', async ({ exec, installedSoftwareOnDisk }) => {
+test('npx playwright install global', async ({ writeConfig, exec, installedSoftwareOnDisk }) => {
   test.skip(process.platform === 'win32', 'isLikelyNpxGlobal() does not work in this setup on our bots');
 
-  const result = await exec('npx playwright install', { env: { npm_config_prefix: '' } }); // global npx and npm_config_prefix do not work together nicely (https://github.com/npm/cli/issues/5268)
+  await writeConfig(true);
+  const result = await exec('npx playwright install');
   expect(result).toHaveLoggedSoftwareDownload(['chromium', 'ffmpeg', 'firefox', 'webkit']);
   expect(await installedSoftwareOnDisk()).toEqual(['chromium', 'ffmpeg', 'firefox', 'webkit']);
   expect(result).not.toContain(`Please run the following command to download new browsers`);

tests/installation/npx-global-spec-codegen.spec.ts

@@ -15,8 +15,9 @@
  */
 import { test, expect } from './npmTest';
 
-test('npx playwright codegen', async ({ exec, installedSoftwareOnDisk }) => {
+test('npx playwright codegen', async ({ writeConfig, exec, installedSoftwareOnDisk }) => {
-  const stdio = await exec('npx playwright codegen', { expectToExitWithError: true, env: { npm_config_prefix: '' } }); // global npx and npm_config_prefix do not work together nicely (https://github.com/npm/cli/issues/5268)
+  await writeConfig(true);
+  const stdio = await exec('npx playwright codegen', { expectToExitWithError: true });
   expect(stdio).toHaveLoggedSoftwareDownload([]);
   expect(await installedSoftwareOnDisk()).toEqual([]);
   expect(stdio).toContain(`Please run the following command to download new browsers`);

tests/installation/playwright-test-should-work.spec.ts

@@ -16,7 +16,7 @@
 import { test } from './npmTest';
 import path from 'path';
 
-test('@playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+test('npm: @playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
   await exec('npm i --foreground-scripts @playwright/test');
   await exec('npx playwright test -c .', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' });
 
@@ -27,3 +27,48 @@ test('@playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspa
   if (nodeMajorVersion >= 14)
     await exec('node', 'esm-playwright-test.mjs');
 });
+
+test('npm: playwright + @playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+  await exec('npm i --foreground-scripts playwright');
+  await exec('npm i --foreground-scripts @playwright/test');
+  await exec('npx playwright install');
+  await exec('npx playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
+  await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
+  await exec('node sanity.js @playwright/test chromium firefox webkit');
+  if (nodeMajorVersion >= 14)
+    await exec('node', 'esm-playwright-test.mjs');
+});
+
+test('npm: @playwright/test + playwright-core should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+  await exec('npm i --foreground-scripts @playwright/test');
+  await exec('npm i --foreground-scripts playwright-core');
+  await exec('npx playwright install');
+  await exec('npx playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
+  await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
+  await exec('node sanity.js @playwright/test chromium firefox webkit');
+  if (nodeMajorVersion >= 14)
+    await exec('node', 'esm-playwright-test.mjs');
+});
+
+test('yarn: @playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+  await exec('yarn add @playwright/test');
+  await exec('yarn playwright test -c .', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' });
+
+  await exec('yarn playwright install');
+  await exec('yarn playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
+  await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
+  await exec('node sanity.js @playwright/test chromium firefox webkit');
+  if (nodeMajorVersion >= 14)
+    await exec('node', 'esm-playwright-test.mjs');
+});
+
+test('pnpm: @playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
+  await exec('pnpm add @playwright/test');
+  await exec('pnpm exec playwright test -c .', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' });
+  await exec('pnpm exec playwright install');
+  await exec('pnpm exec playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
+  await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
+  await exec('node sanity.js @playwright/test chromium firefox webkit');
+  if (nodeMajorVersion >= 14)
+    await exec('node', 'esm-playwright-test.mjs');
+});