test: run installation tests on all major platforms (#13742)

2022-04-26 11:09:49 -07:00 · 2022-04-26 11:09:49 -07:00 · 3e84ab4701
parent c0f0979055
commit 3e84ab4701
16 changed files with 55 additions and 38 deletions
--- a/.github/workflows/tests_secondary.yml
+++ b/.github/workflows/tests_secondary.yml
@ -110,15 +110,23 @@ jobs:
        path: test-results

  test-package-installations:
-    runs-on: ubuntu-20.04
+    name: "Installation Test ${{ matrix.os }} (${{ matrix.node_version }})"
+    runs-on: ${{ matrix.os  }}
    strategy:
      fail-fast: false
      matrix:
+        os:
+        - ubuntu-latest
+        - macos-latest
+        - windows-latest
        node_version:
-        - "^14.1.0"  # pre 14.1, zip extraction was broken (https://github.com/microsoft/playwright/issues/1988)
-        - "^16.0.0"
        - "^18.0.0"
-    timeout-minutes: 20
+        include:
+        - os: ubuntu-latest
+          node_version: "^14.1.0"  # pre 14.1, zip extraction was broken (https://github.com/microsoft/playwright/issues/1988)
+        - os: ubuntu-latest
+          node_version: "^16.0.0"
+    timeout-minutes: 30
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
@ -130,8 +138,10 @@ jobs:
        DEBUG: pw:install
    - run: npm run build
    - run: npx playwright install-deps
-    - name: INSTALLATION TESTS
-      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run itest
+    - run: npm run itest
+      if: matrix.os != 'ubuntu-latest'
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run itest
+      if: matrix.os == 'ubuntu-latest'

  headful_linux:
    name: "Headful Linux"
--- a/tests/installation/connect-to-selenium.spec.ts
+++ b/tests/installation/connect-to-selenium.spec.ts
@ -13,11 +13,14 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
+import os from 'os';
 import path from 'path';
 import { test } from './npmTest';

 test('connect to selenium', async ({ exec, tmpWorkspace }, testInfo) => {
+  test.fixme(os.platform() !== 'linux');
+
  await exec('npm i --foreground-scripts playwright-core');
-  await exec(`node ./download-chromedriver.js ${path.join(tmpWorkspace)}`);
+  await exec(`node download-chromedriver.js ${path.join(tmpWorkspace)}`);
  await exec(`npm run test -- --reporter=list selenium.spec --output=${testInfo.outputPath('tmp-test-results')}`, { cwd: path.join(__dirname, '..', '..'), env: { PWTEST_CHROMEDRIVER: path.join(tmpWorkspace, 'chromedriver') } });
 });
--- a/tests/installation/globalSetup.ts
+++ b/tests/installation/globalSetup.ts
@ -18,14 +18,14 @@ import { spawnAsync } from 'playwright-core/lib/utils/spawnAsync';
 import { rimraf } from 'playwright-core/lib/utilsBundle';
 import { promisify } from 'util';
 import fs from 'fs';
+import { TMP_WORKSPACES } from './npmTest';

 const PACKAGE_BUILDER_SCRIPT = path.join(__dirname, '..', '..', 'utils', 'pack_package.js');
-const TMP_WORKSPACE = '/tmp/pwt/workspaces';

 async function globalSetup() {
-  await promisify(rimraf)(TMP_WORKSPACE);
-  console.log(`Temporary workspaces will be created in ${TMP_WORKSPACE}. They will not be removed at the end. Set DEBUG=itest to determine which sub-dir a specific test is using.`);
-  await fs.promises.mkdir(TMP_WORKSPACE, { recursive: true });
+  await promisify(rimraf)(TMP_WORKSPACES);
+  console.log(`Temporary workspaces will be created in ${TMP_WORKSPACES}. They will not be removed at the end. Set DEBUG=itest to determine which sub-dir a specific test is using.`);
+  await fs.promises.mkdir(TMP_WORKSPACES, { recursive: true });
  if (process.env.PWTEST_INSTALLATION_TEST_SKIP_PACKAGE_BUILDS) {
    console.log('Skipped building packages. Unset PWTEST_INSTALLATION_TEST_SKIP_PACKAGE_BUILDS to build packages.');
    return;
@ -54,7 +54,7 @@ async function globalSetup() {
    build('playwright-webkit'),
  ]);

-  await fs.promises.writeFile(path.join(__dirname, './.registry.json'), JSON.stringify(Object.fromEntries(builds)));
+  await fs.promises.writeFile(path.join(__dirname, '.registry.json'), JSON.stringify(Object.fromEntries(builds)));
 }

 export default globalSetup;
--- a/tests/installation/npm-installs-local-packages.spec.ts
+++ b/tests/installation/npm-installs-local-packages.spec.ts
@ -26,9 +26,9 @@ test('installs local packages', async ({ registry, exec, tmpWorkspace }) => {
  for (const pkg of packages) {
    await test.step(`check version and installation location of ${pkg}`, async () => {
      registry.assertLocalPackage(pkg);
-      const result = await exec('node', `--eval='console.log(JSON.stringify(require.resolve("${pkg}")))'`);
-      const pkgJsonPath = path.join(path.dirname(JSON.parse(result)), 'package.json');
-      expect(pkgJsonPath.startsWith(path.join(tmpWorkspace, 'node_modules'))).toBeTruthy();
+      const result = await exec('node', '--eval', '"console.log(JSON.stringify(require.resolve(process.argv[1])))"', pkg);
+      const pkgJsonPath = fs.realpathSync(path.join(path.dirname(JSON.parse(result)), 'package.json'));
+      expect(pkgJsonPath.startsWith(fs.realpathSync(path.join(tmpWorkspace, 'node_modules')))).toBeTruthy();
      const installedVersion = JSON.parse(await fs.promises.readFile(pkgJsonPath, 'utf8')).version;
      expect(installedVersion).toBe(expectedPlaywrightVersion);
    });
--- a/tests/installation/npmTest.ts
+++ b/tests/installation/npmTest.ts
@ -19,11 +19,15 @@

 import { test as _test, expect as _expect } from '@playwright/test';
 import fs from 'fs';
+import os from 'os';
 import path from 'path';
 import debugLogger from 'debug';
 import { Registry }  from './registry';
 import { spawnAsync } from './spawnAsync';

+
+export const TMP_WORKSPACES = path.join(os.platform() === 'darwin' ? '/tmp' : os.tmpdir(), 'pwt', 'workspaces');
+
 const debug = debugLogger('itest');

 /**
@ -62,7 +66,7 @@ const expect = _expect;
 export type ExecOptions = { cwd?: string, env?: Record<string, string>, message?: string, expectToExitWithError?: boolean };
 export type ArgsOrOptions = [] | [...string[]] | [...string[], ExecOptions] | [ExecOptions];
 export const test = _test.extend<{
-    _autoCopyScripts: void,
+    _auto: void,
    tmpWorkspace: string,
    nodeMajorVersion: number,
    installedSoftwareOnDisk: (registryPath?: string) => Promise<string[]>;
@ -71,11 +75,11 @@ export const test = _test.extend<{
    tsc: (...argsAndOrOptions: ArgsOrOptions) => Promise<string>,
    registry: Registry,
        }>({
-          _autoCopyScripts: [async ({ tmpWorkspace }, use) => {
-            const dstDir = path.join(tmpWorkspace);
+          _auto: [async ({ tmpWorkspace, exec }, use) => {
+            await exec('npm init -y');
            const sourceDir = path.join(__dirname, 'fixture-scripts');
            const contents = await fs.promises.readdir(sourceDir);
-            await Promise.all(contents.map(f => fs.promises.copyFile(path.join(sourceDir, f), path.join(dstDir, f))));
+            await Promise.all(contents.map(f => fs.promises.copyFile(path.join(sourceDir, f), path.join(tmpWorkspace, f))));
            await use();
          }, {
            auto: true,
@ -91,20 +95,16 @@ export const test = _test.extend<{
          },
          tmpWorkspace: async ({}, use) => {
            // We want a location that won't have a node_modules dir anywhere along its path
-            const tmpWorkspace = path.join('/tmp/pwt/workspaces', path.basename(test.info().outputDir));
+            const tmpWorkspace = path.join(TMP_WORKSPACES, path.basename(test.info().outputDir));
            await fs.promises.mkdir(tmpWorkspace);
            debug(`Workspace Folder: ${tmpWorkspace}`);
-            await spawnAsync('npm', ['init', '-y'], {
-              cwd: tmpWorkspace,
-            });
-
            await use(tmpWorkspace);
          },
          registry: async ({}, use, testInfo) => {
            const port = testInfo.workerIndex + 16123;
            const url = `http://127.0.0.1:${port}`;
            const registry = new Registry(testInfo.outputPath('registry'), url);
-            await registry.start(JSON.parse((await fs.promises.readFile(path.join(__dirname, './.registry.json'), 'utf8'))));
+            await registry.start(JSON.parse((await fs.promises.readFile(path.join(__dirname, '.registry.json'), 'utf8'))));
            await use(registry);
            await registry.shutdown();
          },
--- a/tests/installation/playwright-cli-screenshot-should-work.spec.ts
+++ b/tests/installation/playwright-cli-screenshot-should-work.spec.ts
@ -19,7 +19,7 @@ import path from 'path';

 test('playwright cli screenshot should work', async ({ exec, tmpWorkspace }) => {
  await exec('npm i --foreground-scripts playwright');
-  await exec('./node_modules/.bin/playwright screenshot about:blank one.png');
+  await exec(path.join('node_modules', '.bin', 'playwright'), 'screenshot about:blank one.png');
  await fs.promises.stat(path.join(tmpWorkspace, 'one.png'));

  await exec('npx playwright screenshot about:blank two.png');
--- a/tests/installation/playwright-global-installation-cross-package.spec.ts
+++ b/tests/installation/playwright-global-installation-cross-package.spec.ts
@ -24,5 +24,5 @@ test('global installation cross package', async ({ exec, installedSoftwareOnDisk
  expect(await installedSoftwareOnDisk()).toEqual(['chromium', 'ffmpeg', 'firefox', 'webkit']);

  for (const pkg of packages)
-    await test.step(pkg, () => exec('node ./sanity.js', pkg, 'all'));
+    await test.step(pkg, () => exec('node sanity.js', pkg, 'all'));
 });
--- a/tests/installation/playwright-global-installation.spec.ts
+++ b/tests/installation/playwright-global-installation.spec.ts
@ -20,5 +20,5 @@ test('global installation', async ({ exec, installedSoftwareOnDisk }) => {
  expect(result).toHaveLoggedSoftwareDownload(['chromium', 'ffmpeg', 'firefox', 'webkit']);
  expect(await installedSoftwareOnDisk()).toEqual(['chromium', 'ffmpeg', 'firefox', 'webkit']);
  await exec('node sanity.js playwright none', { env: { PLAYWRIGHT_BROWSERS_PATH: undefined } });
-  await exec('node ./sanity.js playwright');
+  await exec('node sanity.js playwright');
 });
--- a/tests/installation/playwright-global-subsequent-installs.spec.ts
+++ b/tests/installation/playwright-global-subsequent-installs.spec.ts
@ -24,5 +24,5 @@ test('subsequent installs works', async ({ exec }) => {
  // sure that script's install.js can be run subsequently without unhandled promise rejections.
  // Note: the flag `--unhandled-rejections=strict` will force node to terminate in case
  // of UnhandledPromiseRejection.
-  await exec('node --unhandled-rejections=strict', path.join('./node_modules', 'playwright', 'install.js'));
+  await exec('node --unhandled-rejections=strict', path.join('node_modules', 'playwright', 'install.js'));
 });
--- a/tests/installation/playwright-should-work-with-relative-browsers-path.spec.ts
+++ b/tests/installation/playwright-should-work-with-relative-browsers-path.spec.ts
@ -20,6 +20,6 @@ import path from 'path';
 test('playwright should work with relative home path', async ({ exec, tmpWorkspace }) => {
  await fs.promises.mkdir(path.join(tmpWorkspace, 'foo'));
  // Make sure that browsers path is resolved relative to the `npm install` call location.
-  await exec('npm i --foreground-scripts playwright', { cwd: path.join(tmpWorkspace, 'foo'), env: { PLAYWRIGHT_BROWSERS_PATH: '../relative' } });
-  await exec('node sanity.js playwright', { env: { PLAYWRIGHT_BROWSERS_PATH: './relative' } });
+  await exec('npm i --foreground-scripts playwright', { cwd: path.join(tmpWorkspace, 'foo'), env: { PLAYWRIGHT_BROWSERS_PATH: path.join('..', 'relative') } });
+  await exec('node sanity.js playwright', { env: { PLAYWRIGHT_BROWSERS_PATH: path.join('.', 'relative') } });
 });
--- a/tests/installation/playwright-should-work-with-relative-home-path.spec.ts
+++ b/tests/installation/playwright-should-work-with-relative-home-path.spec.ts
@ -14,8 +14,11 @@
 * limitations under the License.
 */
 import { test } from './npmTest';
+import os from 'os';

 test('playwright should work with relative home path', async ({ exec }) => {
+  test.skip(os.platform().startsWith('win'));
+
  const env = { PLAYWRIGHT_BROWSERS_PATH: '0', HOME: '.' };
  await exec('npm i --foreground-scripts playwright', { env });
  // Firefox does not work with relative HOME.
--- a/tests/installation/playwright-should-work.spec.ts
+++ b/tests/installation/playwright-should-work.spec.ts
@ -19,7 +19,7 @@ test(`playwright should work`, async ({ exec, nodeMajorVersion, installedSoftwar
  const result = await exec('npm i --foreground-scripts playwright');
  expect(result).toHaveLoggedSoftwareDownload(['chromium', 'ffmpeg', 'firefox', 'webkit']);
  expect(await installedSoftwareOnDisk()).toEqual(['chromium', 'ffmpeg', 'firefox', 'webkit']);
-  await exec('node ./sanity.js playwright');
+  await exec('node sanity.js playwright');
  if (nodeMajorVersion >= 14)
    await exec('node esm-playwright.mjs');
  const stdio = await exec('npx playwright', 'test', '-c', '.', { expectToExitWithError: true });
--- a/tests/installation/playwright-test-should-work.spec.ts
+++ b/tests/installation/playwright-test-should-work.spec.ts
@ -14,14 +14,15 @@
 * limitations under the License.
 */
 import { test } from './npmTest';
+import path from 'path';

-test('@playwright/test should work', async ({ exec, nodeMajorVersion }) => {
+test('@playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => {
  await exec('npm i --foreground-scripts @playwright/test');
  await exec('npx playwright test -c .', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' });

  await exec('npx playwright install');
  await exec('npx playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: {  PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } });
-  await exec('node ./read-json-report.js ./report.json');
+  await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json'));
  await exec('node sanity.js @playwright/test');
  if (nodeMajorVersion >= 14)
    await exec('node', 'esm-playwright-test.mjs');
--- a/tests/installation/playwright-xyz-should-work.spec.ts
+++ b/tests/installation/playwright-xyz-should-work.spec.ts
@ -26,7 +26,7 @@ for (const pkg of ['playwright-chromium', 'playwright-firefox', 'playwright-webk
    expect(result).toHaveLoggedSoftwareDownload(expectedSoftware as any);
    expect(await installedSoftwareOnDisk()).toEqual(expectedSoftware);
    expect(result).not.toContain(`To avoid unexpected behavior, please install your dependencies first`);
-    await exec('node ./sanity.js', pkg);
+    await exec('node sanity.js', pkg);
    if (nodeMajorVersion >= 14)
      await exec('node', `esm-${pkg}.mjs`);
  });
--- a/tests/installation/screencast.spec.ts
+++ b/tests/installation/screencast.spec.ts
@ -19,5 +19,5 @@ test('screencast works', async ({ exec }) => {
  const packages = ['playwright', 'playwright-chromium', 'playwright-firefox', 'playwright-webkit'];
  await exec('npm i --foreground-scripts', ...packages);
  for (const pkg of packages)
-    await test.step(pkg, () => exec('node ./screencast.js', pkg));
+    await test.step(pkg, () => exec('node screencast.js', pkg));
 });
--- a/tests/installation/validate-dependencies.spec.ts
+++ b/tests/installation/validate-dependencies.spec.ts
@ -18,13 +18,13 @@ import { test, expect } from './npmTest';
 test.describe('validate dependencies', () => {
  test('default (on)', async ({ exec }) => {
    await exec('npm i --foreground-scripts playwright');
-    const result = await exec('node ./validate-dependencies.js');
+    const result = await exec('node validate-dependencies.js');
    expect(result).toContain(`PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS`);
  });

  test('disabled (off)',  async ({ exec }) => {
    await exec('npm i --foreground-scripts playwright');
-    const result = await exec('node ./validate-dependencies-skip-executable-path.js');
+    const result = await exec('node validate-dependencies-skip-executable-path.js');
    expect(result).not.toContain(`PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS`);
  });
 });