diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 0d1cdbc4a4..ff5ca9c175 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -110,15 +110,23 @@ jobs: path: test-results test-package-installations: - runs-on: ubuntu-20.04 + name: "Installation Test ${{ matrix.os }} (${{ matrix.node_version }})" + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest node_version: - - "^14.1.0" # pre 14.1, zip extraction was broken (https://github.com/microsoft/playwright/issues/1988) - - "^16.0.0" - "^18.0.0" - timeout-minutes: 20 + include: + - os: ubuntu-latest + node_version: "^14.1.0" # pre 14.1, zip extraction was broken (https://github.com/microsoft/playwright/issues/1988) + - os: ubuntu-latest + node_version: "^16.0.0" + timeout-minutes: 30 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 @@ -130,8 +138,10 @@ jobs: DEBUG: pw:install - run: npm run build - run: npx playwright install-deps - - name: INSTALLATION TESTS - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run itest + - run: npm run itest + if: matrix.os != 'ubuntu-latest' + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run itest + if: matrix.os == 'ubuntu-latest' headful_linux: name: "Headful Linux" diff --git a/tests/installation/connect-to-selenium.spec.ts b/tests/installation/connect-to-selenium.spec.ts index df4866822e..a155f1b090 100755 --- a/tests/installation/connect-to-selenium.spec.ts +++ b/tests/installation/connect-to-selenium.spec.ts @@ -13,11 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import os from 'os'; import path from 'path'; import { test } from './npmTest'; test('connect to selenium', async ({ exec, tmpWorkspace }, testInfo) => { + test.fixme(os.platform() !== 'linux'); + await exec('npm i --foreground-scripts playwright-core'); - await exec(`node ./download-chromedriver.js ${path.join(tmpWorkspace)}`); + await exec(`node download-chromedriver.js ${path.join(tmpWorkspace)}`); await exec(`npm run test -- --reporter=list selenium.spec --output=${testInfo.outputPath('tmp-test-results')}`, { cwd: path.join(__dirname, '..', '..'), env: { PWTEST_CHROMEDRIVER: path.join(tmpWorkspace, 'chromedriver') } }); }); diff --git a/tests/installation/globalSetup.ts b/tests/installation/globalSetup.ts index 3c84815980..936ecb0b13 100644 --- a/tests/installation/globalSetup.ts +++ b/tests/installation/globalSetup.ts @@ -18,14 +18,14 @@ import { spawnAsync } from 'playwright-core/lib/utils/spawnAsync'; import { rimraf } from 'playwright-core/lib/utilsBundle'; import { promisify } from 'util'; import fs from 'fs'; +import { TMP_WORKSPACES } from './npmTest'; const PACKAGE_BUILDER_SCRIPT = path.join(__dirname, '..', '..', 'utils', 'pack_package.js'); -const TMP_WORKSPACE = '/tmp/pwt/workspaces'; async function globalSetup() { - await promisify(rimraf)(TMP_WORKSPACE); - console.log(`Temporary workspaces will be created in ${TMP_WORKSPACE}. They will not be removed at the end. Set DEBUG=itest to determine which sub-dir a specific test is using.`); - await fs.promises.mkdir(TMP_WORKSPACE, { recursive: true }); + await promisify(rimraf)(TMP_WORKSPACES); + console.log(`Temporary workspaces will be created in ${TMP_WORKSPACES}. They will not be removed at the end. Set DEBUG=itest to determine which sub-dir a specific test is using.`); + await fs.promises.mkdir(TMP_WORKSPACES, { recursive: true }); if (process.env.PWTEST_INSTALLATION_TEST_SKIP_PACKAGE_BUILDS) { console.log('Skipped building packages. Unset PWTEST_INSTALLATION_TEST_SKIP_PACKAGE_BUILDS to build packages.'); return; @@ -54,7 +54,7 @@ async function globalSetup() { build('playwright-webkit'), ]); - await fs.promises.writeFile(path.join(__dirname, './.registry.json'), JSON.stringify(Object.fromEntries(builds))); + await fs.promises.writeFile(path.join(__dirname, '.registry.json'), JSON.stringify(Object.fromEntries(builds))); } export default globalSetup; diff --git a/tests/installation/npm-installs-local-packages.spec.ts b/tests/installation/npm-installs-local-packages.spec.ts index f40793b5e8..1645d805bb 100755 --- a/tests/installation/npm-installs-local-packages.spec.ts +++ b/tests/installation/npm-installs-local-packages.spec.ts @@ -26,9 +26,9 @@ test('installs local packages', async ({ registry, exec, tmpWorkspace }) => { for (const pkg of packages) { await test.step(`check version and installation location of ${pkg}`, async () => { registry.assertLocalPackage(pkg); - const result = await exec('node', `--eval='console.log(JSON.stringify(require.resolve("${pkg}")))'`); - const pkgJsonPath = path.join(path.dirname(JSON.parse(result)), 'package.json'); - expect(pkgJsonPath.startsWith(path.join(tmpWorkspace, 'node_modules'))).toBeTruthy(); + const result = await exec('node', '--eval', '"console.log(JSON.stringify(require.resolve(process.argv[1])))"', pkg); + const pkgJsonPath = fs.realpathSync(path.join(path.dirname(JSON.parse(result)), 'package.json')); + expect(pkgJsonPath.startsWith(fs.realpathSync(path.join(tmpWorkspace, 'node_modules')))).toBeTruthy(); const installedVersion = JSON.parse(await fs.promises.readFile(pkgJsonPath, 'utf8')).version; expect(installedVersion).toBe(expectedPlaywrightVersion); }); diff --git a/tests/installation/npmTest.ts b/tests/installation/npmTest.ts index 8970459640..2d802d8700 100644 --- a/tests/installation/npmTest.ts +++ b/tests/installation/npmTest.ts @@ -19,11 +19,15 @@ import { test as _test, expect as _expect } from '@playwright/test'; import fs from 'fs'; +import os from 'os'; import path from 'path'; import debugLogger from 'debug'; import { Registry } from './registry'; import { spawnAsync } from './spawnAsync'; + +export const TMP_WORKSPACES = path.join(os.platform() === 'darwin' ? '/tmp' : os.tmpdir(), 'pwt', 'workspaces'); + const debug = debugLogger('itest'); /** @@ -62,7 +66,7 @@ const expect = _expect; export type ExecOptions = { cwd?: string, env?: Record, message?: string, expectToExitWithError?: boolean }; export type ArgsOrOptions = [] | [...string[]] | [...string[], ExecOptions] | [ExecOptions]; export const test = _test.extend<{ - _autoCopyScripts: void, + _auto: void, tmpWorkspace: string, nodeMajorVersion: number, installedSoftwareOnDisk: (registryPath?: string) => Promise; @@ -71,11 +75,11 @@ export const test = _test.extend<{ tsc: (...argsAndOrOptions: ArgsOrOptions) => Promise, registry: Registry, }>({ - _autoCopyScripts: [async ({ tmpWorkspace }, use) => { - const dstDir = path.join(tmpWorkspace); + _auto: [async ({ tmpWorkspace, exec }, use) => { + await exec('npm init -y'); const sourceDir = path.join(__dirname, 'fixture-scripts'); const contents = await fs.promises.readdir(sourceDir); - await Promise.all(contents.map(f => fs.promises.copyFile(path.join(sourceDir, f), path.join(dstDir, f)))); + await Promise.all(contents.map(f => fs.promises.copyFile(path.join(sourceDir, f), path.join(tmpWorkspace, f)))); await use(); }, { auto: true, @@ -91,20 +95,16 @@ export const test = _test.extend<{ }, tmpWorkspace: async ({}, use) => { // We want a location that won't have a node_modules dir anywhere along its path - const tmpWorkspace = path.join('/tmp/pwt/workspaces', path.basename(test.info().outputDir)); + const tmpWorkspace = path.join(TMP_WORKSPACES, path.basename(test.info().outputDir)); await fs.promises.mkdir(tmpWorkspace); debug(`Workspace Folder: ${tmpWorkspace}`); - await spawnAsync('npm', ['init', '-y'], { - cwd: tmpWorkspace, - }); - await use(tmpWorkspace); }, registry: async ({}, use, testInfo) => { const port = testInfo.workerIndex + 16123; const url = `http://127.0.0.1:${port}`; const registry = new Registry(testInfo.outputPath('registry'), url); - await registry.start(JSON.parse((await fs.promises.readFile(path.join(__dirname, './.registry.json'), 'utf8')))); + await registry.start(JSON.parse((await fs.promises.readFile(path.join(__dirname, '.registry.json'), 'utf8')))); await use(registry); await registry.shutdown(); }, diff --git a/tests/installation/playwright-cli-screenshot-should-work.spec.ts b/tests/installation/playwright-cli-screenshot-should-work.spec.ts index 7a448a101a..5cbcb486f8 100755 --- a/tests/installation/playwright-cli-screenshot-should-work.spec.ts +++ b/tests/installation/playwright-cli-screenshot-should-work.spec.ts @@ -19,7 +19,7 @@ import path from 'path'; test('playwright cli screenshot should work', async ({ exec, tmpWorkspace }) => { await exec('npm i --foreground-scripts playwright'); - await exec('./node_modules/.bin/playwright screenshot about:blank one.png'); + await exec(path.join('node_modules', '.bin', 'playwright'), 'screenshot about:blank one.png'); await fs.promises.stat(path.join(tmpWorkspace, 'one.png')); await exec('npx playwright screenshot about:blank two.png'); diff --git a/tests/installation/playwright-global-installation-cross-package.spec.ts b/tests/installation/playwright-global-installation-cross-package.spec.ts index 723a11132d..2a002b7969 100755 --- a/tests/installation/playwright-global-installation-cross-package.spec.ts +++ b/tests/installation/playwright-global-installation-cross-package.spec.ts @@ -24,5 +24,5 @@ test('global installation cross package', async ({ exec, installedSoftwareOnDisk expect(await installedSoftwareOnDisk()).toEqual(['chromium', 'ffmpeg', 'firefox', 'webkit']); for (const pkg of packages) - await test.step(pkg, () => exec('node ./sanity.js', pkg, 'all')); + await test.step(pkg, () => exec('node sanity.js', pkg, 'all')); }); diff --git a/tests/installation/playwright-global-installation.spec.ts b/tests/installation/playwright-global-installation.spec.ts index b8805e8128..b51c92b6c0 100755 --- a/tests/installation/playwright-global-installation.spec.ts +++ b/tests/installation/playwright-global-installation.spec.ts @@ -20,5 +20,5 @@ test('global installation', async ({ exec, installedSoftwareOnDisk }) => { expect(result).toHaveLoggedSoftwareDownload(['chromium', 'ffmpeg', 'firefox', 'webkit']); expect(await installedSoftwareOnDisk()).toEqual(['chromium', 'ffmpeg', 'firefox', 'webkit']); await exec('node sanity.js playwright none', { env: { PLAYWRIGHT_BROWSERS_PATH: undefined } }); - await exec('node ./sanity.js playwright'); + await exec('node sanity.js playwright'); }); diff --git a/tests/installation/playwright-global-subsequent-installs.spec.ts b/tests/installation/playwright-global-subsequent-installs.spec.ts index 82354a595c..4db5ac1704 100755 --- a/tests/installation/playwright-global-subsequent-installs.spec.ts +++ b/tests/installation/playwright-global-subsequent-installs.spec.ts @@ -24,5 +24,5 @@ test('subsequent installs works', async ({ exec }) => { // sure that script's install.js can be run subsequently without unhandled promise rejections. // Note: the flag `--unhandled-rejections=strict` will force node to terminate in case // of UnhandledPromiseRejection. - await exec('node --unhandled-rejections=strict', path.join('./node_modules', 'playwright', 'install.js')); + await exec('node --unhandled-rejections=strict', path.join('node_modules', 'playwright', 'install.js')); }); diff --git a/tests/installation/playwright-should-work-with-relative-browsers-path.spec.ts b/tests/installation/playwright-should-work-with-relative-browsers-path.spec.ts index 7d63f88c1d..9d6388f21b 100755 --- a/tests/installation/playwright-should-work-with-relative-browsers-path.spec.ts +++ b/tests/installation/playwright-should-work-with-relative-browsers-path.spec.ts @@ -20,6 +20,6 @@ import path from 'path'; test('playwright should work with relative home path', async ({ exec, tmpWorkspace }) => { await fs.promises.mkdir(path.join(tmpWorkspace, 'foo')); // Make sure that browsers path is resolved relative to the `npm install` call location. - await exec('npm i --foreground-scripts playwright', { cwd: path.join(tmpWorkspace, 'foo'), env: { PLAYWRIGHT_BROWSERS_PATH: '../relative' } }); - await exec('node sanity.js playwright', { env: { PLAYWRIGHT_BROWSERS_PATH: './relative' } }); + await exec('npm i --foreground-scripts playwright', { cwd: path.join(tmpWorkspace, 'foo'), env: { PLAYWRIGHT_BROWSERS_PATH: path.join('..', 'relative') } }); + await exec('node sanity.js playwright', { env: { PLAYWRIGHT_BROWSERS_PATH: path.join('.', 'relative') } }); }); diff --git a/tests/installation/playwright-should-work-with-relative-home-path.spec.ts b/tests/installation/playwright-should-work-with-relative-home-path.spec.ts index a19b51b4b2..6e4daac128 100755 --- a/tests/installation/playwright-should-work-with-relative-home-path.spec.ts +++ b/tests/installation/playwright-should-work-with-relative-home-path.spec.ts @@ -14,8 +14,11 @@ * limitations under the License. */ import { test } from './npmTest'; +import os from 'os'; test('playwright should work with relative home path', async ({ exec }) => { + test.skip(os.platform().startsWith('win')); + const env = { PLAYWRIGHT_BROWSERS_PATH: '0', HOME: '.' }; await exec('npm i --foreground-scripts playwright', { env }); // Firefox does not work with relative HOME. diff --git a/tests/installation/playwright-should-work.spec.ts b/tests/installation/playwright-should-work.spec.ts index eb3ff4fb3c..80a4376941 100755 --- a/tests/installation/playwright-should-work.spec.ts +++ b/tests/installation/playwright-should-work.spec.ts @@ -19,7 +19,7 @@ test(`playwright should work`, async ({ exec, nodeMajorVersion, installedSoftwar const result = await exec('npm i --foreground-scripts playwright'); expect(result).toHaveLoggedSoftwareDownload(['chromium', 'ffmpeg', 'firefox', 'webkit']); expect(await installedSoftwareOnDisk()).toEqual(['chromium', 'ffmpeg', 'firefox', 'webkit']); - await exec('node ./sanity.js playwright'); + await exec('node sanity.js playwright'); if (nodeMajorVersion >= 14) await exec('node esm-playwright.mjs'); const stdio = await exec('npx playwright', 'test', '-c', '.', { expectToExitWithError: true }); diff --git a/tests/installation/playwright-test-should-work.spec.ts b/tests/installation/playwright-test-should-work.spec.ts index 1e64d4a241..bacbc0a247 100755 --- a/tests/installation/playwright-test-should-work.spec.ts +++ b/tests/installation/playwright-test-should-work.spec.ts @@ -14,14 +14,15 @@ * limitations under the License. */ import { test } from './npmTest'; +import path from 'path'; -test('@playwright/test should work', async ({ exec, nodeMajorVersion }) => { +test('@playwright/test should work', async ({ exec, nodeMajorVersion, tmpWorkspace }) => { await exec('npm i --foreground-scripts @playwright/test'); await exec('npx playwright test -c .', { expectToExitWithError: true, message: 'should not be able to run tests without installing browsers' }); await exec('npx playwright install'); await exec('npx playwright test -c . --browser=all --reporter=list,json sample.spec.js', { env: { PLAYWRIGHT_JSON_OUTPUT_NAME: 'report.json' } }); - await exec('node ./read-json-report.js ./report.json'); + await exec('node read-json-report.js', path.join(tmpWorkspace, 'report.json')); await exec('node sanity.js @playwright/test'); if (nodeMajorVersion >= 14) await exec('node', 'esm-playwright-test.mjs'); diff --git a/tests/installation/playwright-xyz-should-work.spec.ts b/tests/installation/playwright-xyz-should-work.spec.ts index 945413f577..a5a888e920 100755 --- a/tests/installation/playwright-xyz-should-work.spec.ts +++ b/tests/installation/playwright-xyz-should-work.spec.ts @@ -26,7 +26,7 @@ for (const pkg of ['playwright-chromium', 'playwright-firefox', 'playwright-webk expect(result).toHaveLoggedSoftwareDownload(expectedSoftware as any); expect(await installedSoftwareOnDisk()).toEqual(expectedSoftware); expect(result).not.toContain(`To avoid unexpected behavior, please install your dependencies first`); - await exec('node ./sanity.js', pkg); + await exec('node sanity.js', pkg); if (nodeMajorVersion >= 14) await exec('node', `esm-${pkg}.mjs`); }); diff --git a/tests/installation/screencast.spec.ts b/tests/installation/screencast.spec.ts index 84060ed535..5913b5a125 100755 --- a/tests/installation/screencast.spec.ts +++ b/tests/installation/screencast.spec.ts @@ -19,5 +19,5 @@ test('screencast works', async ({ exec }) => { const packages = ['playwright', 'playwright-chromium', 'playwright-firefox', 'playwright-webkit']; await exec('npm i --foreground-scripts', ...packages); for (const pkg of packages) - await test.step(pkg, () => exec('node ./screencast.js', pkg)); + await test.step(pkg, () => exec('node screencast.js', pkg)); }); diff --git a/tests/installation/validate-dependencies.spec.ts b/tests/installation/validate-dependencies.spec.ts index 9ca7e71af1..f8a56b7d48 100644 --- a/tests/installation/validate-dependencies.spec.ts +++ b/tests/installation/validate-dependencies.spec.ts @@ -18,13 +18,13 @@ import { test, expect } from './npmTest'; test.describe('validate dependencies', () => { test('default (on)', async ({ exec }) => { await exec('npm i --foreground-scripts playwright'); - const result = await exec('node ./validate-dependencies.js'); + const result = await exec('node validate-dependencies.js'); expect(result).toContain(`PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS`); }); test('disabled (off)', async ({ exec }) => { await exec('npm i --foreground-scripts playwright'); - const result = await exec('node ./validate-dependencies-skip-executable-path.js'); + const result = await exec('node validate-dependencies-skip-executable-path.js'); expect(result).not.toContain(`PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS`); }); });