From 2c59985bcdd35cdaca75a74578ab3f2ab7009453 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Fri, 11 Feb 2022 18:06:37 -0700 Subject: [PATCH] chore: refactor installation tests (#12054) This patch refactors installation tests. With this refactoring: - each test is a separate file - to run a single test, just run the bash file * tests support optional `--no-build` flag to re-use previously built packages. * tests support optional `--debug` flag to see line-by-line test output * failed tests print a line that can be copied-and-pasted locally to debug the test - run all tests with `//installation-tests/run_all_tests.sh` * test output is hidden for successful runs when run locally and is shown when test fails * runs all tests, and reports failed tests in the end * command output is split into groups when viewed on Github. --- .github/workflows/tests_secondary.yml | 3 +- installation-tests/initialize_test.sh | 130 ++++ installation-tests/installation-tests.sh | 689 ------------------ installation-tests/run_all_tests.sh | 54 ++ installation-tests/test_android_types.sh | 11 + .../test_connect_to_selenium.sh | 6 + installation-tests/test_electron_types.sh | 13 + .../test_playwright_chromium_should_work.sh | 25 + ...test_playwright_cli_codegen_should_work.sh | 38 + ...test_playwright_cli_install_should_work.sh | 51 ++ ...t_playwright_cli_screenshot_should_work.sh | 19 + .../test_playwright_driver_should_work.sh | 13 + .../test_playwright_electron_should_work.sh | 10 + .../test_playwright_firefox_should_work.sh | 25 + .../test_playwright_global_installation.sh | 16 + ...right_global_installation_cross_package.sh | 22 + ...global_installation_subsequent_installs.sh | 17 + .../test_playwright_should_work.sh | 32 + ...should_work_with_relative_browsers_path.sh | 14 + ...ght_should_work_with_relative_home_path.sh | 9 + .../test_playwright_test_should_work.sh | 29 + ...test_playwright_test_stacks_should_work.sh | 18 + .../test_playwright_validate_dependencies.sh | 13 + ...idate_dependencies_skip_executable_path.sh | 13 + .../test_playwright_webkit_should_work.sh | 25 + installation-tests/test_screencast.sh | 18 + .../test_skip_browser_download.sh | 15 + ...download_inspect_with_custom_executable.sh | 25 + installation-tests/test_typescript_types.sh | 30 + 29 files changed, 693 insertions(+), 690 deletions(-) create mode 100755 installation-tests/initialize_test.sh delete mode 100755 installation-tests/installation-tests.sh create mode 100755 installation-tests/run_all_tests.sh create mode 100644 installation-tests/test_android_types.sh create mode 100644 installation-tests/test_connect_to_selenium.sh create mode 100644 installation-tests/test_electron_types.sh create mode 100644 installation-tests/test_playwright_chromium_should_work.sh create mode 100644 installation-tests/test_playwright_cli_codegen_should_work.sh create mode 100644 installation-tests/test_playwright_cli_install_should_work.sh create mode 100644 installation-tests/test_playwright_cli_screenshot_should_work.sh create mode 100644 installation-tests/test_playwright_driver_should_work.sh create mode 100644 installation-tests/test_playwright_electron_should_work.sh create mode 100644 installation-tests/test_playwright_firefox_should_work.sh create mode 100644 installation-tests/test_playwright_global_installation.sh create mode 100644 installation-tests/test_playwright_global_installation_cross_package.sh create mode 100644 installation-tests/test_playwright_global_installation_subsequent_installs.sh create mode 100644 installation-tests/test_playwright_should_work.sh create mode 100644 installation-tests/test_playwright_should_work_with_relative_browsers_path.sh create mode 100644 installation-tests/test_playwright_should_work_with_relative_home_path.sh create mode 100644 installation-tests/test_playwright_test_should_work.sh create mode 100644 installation-tests/test_playwright_test_stacks_should_work.sh create mode 100644 installation-tests/test_playwright_validate_dependencies.sh create mode 100644 installation-tests/test_playwright_validate_dependencies_skip_executable_path.sh create mode 100644 installation-tests/test_playwright_webkit_should_work.sh create mode 100644 installation-tests/test_screencast.sh create mode 100644 installation-tests/test_skip_browser_download.sh create mode 100644 installation-tests/test_skip_browser_download_inspect_with_custom_executable.sh create mode 100644 installation-tests/test_typescript_types.sh diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 2ee1bfcafc..4607cf4926 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -132,7 +132,8 @@ jobs: DEBUG: pw:install - run: npm run build - run: npx playwright install-deps - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash installation-tests/installation-tests.sh + - name: INSTALLATION TESTS + run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash installation-tests/run_all_tests.sh headful_linux: name: "Headful Linux" diff --git a/installation-tests/initialize_test.sh b/installation-tests/initialize_test.sh new file mode 100755 index 0000000000..c403c3bd08 --- /dev/null +++ b/installation-tests/initialize_test.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# break script execution if some command returns non-zero exit code +set -e + +function build_packages() { + local PACKAGE_BUILDER="../../utils/pack_package.js" + rm -rf ./output + mkdir ./output + pushd ./output >/dev/null + + node ${PACKAGE_BUILDER} playwright-core "${PLAYWRIGHT_CORE_TGZ}" 2>&1 1>/dev/null + node ${PACKAGE_BUILDER} playwright-test "${PLAYWRIGHT_TEST_TGZ}" 2>&1 1>/dev/null + node ${PACKAGE_BUILDER} playwright "${PLAYWRIGHT_TGZ}" 2>&1 1>/dev/null + node ${PACKAGE_BUILDER} playwright-chromium "${PLAYWRIGHT_CHROMIUM_TGZ}" 2>&1 1>/dev/null + node ${PACKAGE_BUILDER} playwright-webkit "${PLAYWRIGHT_WEBKIT_TGZ}" 2>&1 1>/dev/null + node ${PACKAGE_BUILDER} playwright-firefox "${PLAYWRIGHT_FIREFOX_TGZ}" 2>&1 1>/dev/null + popd >/dev/null +} + +function cecho(){ + local RED="\033[0;31m" + local GREEN="\033[0;32m" + local YELLOW="\033[1;33m" + local NC="\033[0m" # No Color + printf "${!1}${2} ${NC}\n" +} + +function complete_test { + set +x + if [[ $? == 0 ]]; then + echo + cecho "GREEN" "<<<<<<<<<<<<" + cecho "GREEN" " Test '${TEST_FILE}' PASSED" + cecho "GREEN" "<<<<<<<<<<<<" + else + cecho "RED" "<<<<<<<<<<<<" + cecho "RED" " Test '${TEST_FILE}' FAILED" + cecho "RED" " To debug locally, run:" + cecho "RED" " bash ${TEST_PATH} --debug" + cecho "RED" "<<<<<<<<<<<<" + echo + fi + echo +} + +function setup_env_variables() { + # Package paths. + SCRIPTS_PATH="$(pwd -P)" + TEST_ROOT="/tmp/playwright-installation-tests" + NODE_VERSION=$(node -e "console.log(process.version.slice(1).split('.')[0])") + + PLAYWRIGHT_CORE_TGZ="${PWD}/output/playwright-core.tgz" + PLAYWRIGHT_TGZ="${PWD}/output/playwright.tgz" + PLAYWRIGHT_CHROMIUM_TGZ="${PWD}/output/playwright-chromium.tgz" + PLAYWRIGHT_WEBKIT_TGZ="${PWD}/output/playwright-webkit.tgz" + PLAYWRIGHT_FIREFOX_TGZ="${PWD}/output/playwright-firefox.tgz" + PLAYWRIGHT_TEST_TGZ="${PWD}/output/playwright-test.tgz" +} + +function clean_test_root() { + rm -rf "${TEST_ROOT}" + mkdir -p "${TEST_ROOT}" +} + +function initialize_test { + trap "complete_test;cd $(pwd -P)" EXIT + cd "$(dirname $0)" + + # cleanup environment + unset PLAYWRIGHT_DOWNLOAD_HOST + unset PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD + export PLAYWRIGHT_BROWSERS_PATH=0 + + setup_env_variables + + if [[ "$1" != "--no-build" && "$2" != "--no-build" ]]; then + echo 'Building packages... NOTE: run with `--no-build` to reuse previous builds' + build_packages + else + if [[ ! -f "${PLAYWRIGHT_TGZ}" || \ + ! -f "${PLAYWRIGHT_CORE_TGZ}" || \ + ! -f "${PLAYWRIGHT_CHROMIUM_TGZ}" || \ + ! -f "${PLAYWRIGHT_WEBKIT_TGZ}" || \ + ! -f "${PLAYWRIGHT_FIREFOX_TGZ}" || \ + ! -f "${PLAYWRIGHT_TEST_TGZ}" ]]; then + echo 'ERROR: cannot run test with `--no-build` flag! One of the packages is missing!' + exit 1 + fi + fi + if [[ "$1" != "--do-not-clean-test-root" && "$2" != "--do-not-clean-test-root" ]]; then + clean_test_root + fi + cd ${TEST_ROOT} + TEST_FILE=$(basename $0) + TEST_NAME=$(basename ${0%%.sh}) + TEST_PATH="${PWD}/${TEST_FILE}" + + cecho "YELLOW" ">>>>>>>>>>>>" + cecho "YELLOW" " Running test - '${TEST_FILE}'" + cecho "YELLOW" ">>>>>>>>>>>>" + mkdir ${TEST_NAME} && cd ${TEST_NAME} && npm init -y 1>/dev/null 2>/dev/null + + # enable bash lines logging if --debug is passed + if [[ $1 == "--debug" || $2 == "--debug" ]]; then + set -x + fi +} + +function copy_test_scripts { + cp "${SCRIPTS_PATH}/inspector-custom-executable.js" . + cp "${SCRIPTS_PATH}/sanity.js" . + cp "${SCRIPTS_PATH}/screencast.js" . + cp "${SCRIPTS_PATH}/validate-dependencies.js" . + cp "${SCRIPTS_PATH}/validate-dependencies-skip-executable-path.js" . + cp "${SCRIPTS_PATH}/esm.mjs" . + cp "${SCRIPTS_PATH}/esm-playwright.mjs" . + cp "${SCRIPTS_PATH}/esm-playwright-chromium.mjs" . + cp "${SCRIPTS_PATH}/esm-playwright-firefox.mjs" . + cp "${SCRIPTS_PATH}/esm-playwright-webkit.mjs" . + cp "${SCRIPTS_PATH}/esm-playwright-test.mjs" . + cp "${SCRIPTS_PATH}/sanity-electron.js" . + cp "${SCRIPTS_PATH}/electron-app.js" . + cp "${SCRIPTS_PATH}/driver-client.js" . + cp "${SCRIPTS_PATH}/sample.spec.js" . + cp "${SCRIPTS_PATH}/failing.spec.js" . + cp "${SCRIPTS_PATH}/read-json-report.js" . + cp "${SCRIPTS_PATH}/playwright-test-types.ts" . +} + diff --git a/installation-tests/installation-tests.sh b/installation-tests/installation-tests.sh deleted file mode 100755 index 7364cd400f..0000000000 --- a/installation-tests/installation-tests.sh +++ /dev/null @@ -1,689 +0,0 @@ -#!/bin/bash -set -e -set +x - -trap "cd $(pwd -P)" EXIT -cd "$(dirname $0)" - -rm -rf ./output -mkdir ./output -cd ./output - -# cleanup environment -unset PLAYWRIGHT_DOWNLOAD_HOST -unset PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD -export PLAYWRIGHT_BROWSERS_PATH=0 - -# Pack all packages and put them in our output folder. -echo "Building packages..." -PACKAGE_BUILDER="../../utils/pack_package.js" -PLAYWRIGHT_CORE_TGZ="$(node ${PACKAGE_BUILDER} playwright-core ./playwright-core.tgz)" -echo "playwright-core built" -PLAYWRIGHT_TGZ="$(node ${PACKAGE_BUILDER} playwright ./playwright.tgz)" -echo "playwright built" -PLAYWRIGHT_CHROMIUM_TGZ="$(node ${PACKAGE_BUILDER} playwright-chromium ./playwright-chromium.tgz)" -echo "playwright-chromium built" -PLAYWRIGHT_WEBKIT_TGZ="$(node ${PACKAGE_BUILDER} playwright-webkit ./playwright-webkit.tgz)" -echo "playwright-webkit built" -PLAYWRIGHT_FIREFOX_TGZ="$(node ${PACKAGE_BUILDER} playwright-firefox ./playwright-firefox.tgz)" -echo "playwright-firefox built" -PLAYWRIGHT_TEST_TGZ="$(node ${PACKAGE_BUILDER} playwright-test ./playwright-test.tgz)" -echo "playwright-test built" - -SCRIPTS_PATH="$(pwd -P)/.." -TEST_ROOT="/tmp/playwright-installation-tests" -rm -rf "${TEST_ROOT}" -mkdir -p "${TEST_ROOT}" -NODE_VERSION=$(node -e "console.log(process.version.slice(1).split('.')[0])") - -function copy_test_scripts { - cp "${SCRIPTS_PATH}/inspector-custom-executable.js" . - cp "${SCRIPTS_PATH}/sanity.js" . - cp "${SCRIPTS_PATH}/screencast.js" . - cp "${SCRIPTS_PATH}/validate-dependencies.js" . - cp "${SCRIPTS_PATH}/validate-dependencies-skip-executable-path.js" . - cp "${SCRIPTS_PATH}/esm.mjs" . - cp "${SCRIPTS_PATH}/esm-playwright.mjs" . - cp "${SCRIPTS_PATH}/esm-playwright-chromium.mjs" . - cp "${SCRIPTS_PATH}/esm-playwright-firefox.mjs" . - cp "${SCRIPTS_PATH}/esm-playwright-webkit.mjs" . - cp "${SCRIPTS_PATH}/esm-playwright-test.mjs" . - cp "${SCRIPTS_PATH}/sanity-electron.js" . - cp "${SCRIPTS_PATH}/electron-app.js" . - cp "${SCRIPTS_PATH}/driver-client.js" . - cp "${SCRIPTS_PATH}/sample.spec.js" . - cp "${SCRIPTS_PATH}/failing.spec.js" . - cp "${SCRIPTS_PATH}/read-json-report.js" . - cp "${SCRIPTS_PATH}/playwright-test-types.ts" . -} - -function run_tests { - test_playwright_test_should_work - test_connect_to_selenium - test_screencast - test_typescript_types - test_playwright_global_installation_subsequent_installs - test_playwright_should_work_with_relative_home_path - test_playwright_should_work_with_relative_browsers_path - test_playwright_validate_dependencies - test_playwright_validate_dependencies_skip_executable_path - test_playwright_global_installation - test_playwright_global_installation_cross_package - test_playwright_electron_should_work - test_electron_types - test_android_types - test_playwright_cli_screenshot_should_work - test_playwright_cli_install_should_work - test_playwright_cli_codegen_should_work - test_playwright_driver_should_work - test_skip_browser_download - test_skip_browser_download_inspect_with_custom_executable - test_playwright_should_work - test_playwright_chromium_should_work - test_playwright_webkit_should_work - test_playwright_firefox_should_work - test_playwright_test_stacks_should_work -} - -function test_screencast { - initialize_test "${FUNCNAME[0]}" - copy_test_scripts - - local BROWSERS="$(pwd -P)/browsers" - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_TGZ} - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_FIREFOX_TGZ} - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_WEBKIT_TGZ} - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_CHROMIUM_TGZ} - - echo "Running screencast.js" - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node screencast.js playwright - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node screencast.js playwright-chromium - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node screencast.js playwright-webkit - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node screencast.js playwright-firefox - - echo "${FUNCNAME[0]} success" -} - -function test_typescript_types { - initialize_test "${FUNCNAME[0]}" - copy_test_scripts - # @types/node@14.18.9 is the last version which is compatibel with typescript@3.7.5. - # After @types/node@14.18.9 URLSearchParams from @types/node conflicts with typescript's - # shipped types and it results in a type error / build failure. - npm install -D @types/node@14.18.9 - - # install all packages. - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_FIREFOX_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_WEBKIT_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_CHROMIUM_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TEST_TGZ} - - # typecheck all packages. - for PKG_NAME in "playwright" \ - "playwright-core" \ - "playwright-firefox" \ - "playwright-chromium" \ - "playwright-webkit" - do - echo "Checking types of ${PKG_NAME}" - echo "import { Page } from '${PKG_NAME}';" > "${PKG_NAME}.ts" && npx -p typescript@3.7.5 tsc "${PKG_NAME}.ts" - done; - - echo "Checking types of @playwright/test" - echo npx -p typescript@3.7.5 tsc "playwright-test-types.ts" - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_global_installation { - initialize_test "${FUNCNAME[0]}" - - local BROWSERS="$(pwd -P)/browsers" - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_TGZ} - if [[ ! -d "${BROWSERS}" ]]; then - echo "Directory for shared browsers was not created!" - exit 1 - fi - copy_test_scripts - - echo "Running sanity.js" - node sanity.js playwright none - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_global_installation_cross_package { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_FIREFOX_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_WEBKIT_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_CHROMIUM_TGZ} - - local BROWSERS="$(pwd -P)/browsers" - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_TGZ} - if [[ ! -d "${BROWSERS}" ]]; then - echo "Directory for shared browsers was not created!" - exit 1 - fi - - copy_test_scripts - - echo "Running sanity.js" - # Every package should be able to launch. - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright-chromium all - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright-firefox all - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright-webkit all - - echo "${FUNCNAME[0]} success" -} - -# @see https://github.com/microsoft/playwright/issues/1651 -function test_playwright_global_installation_subsequent_installs { - initialize_test "${FUNCNAME[0]}" - - local BROWSERS="$(pwd -P)/browsers" - - mkdir install-1 && pushd install-1 && npm init -y - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_TGZ} - # Note: the `npm install` would not actually crash, the error - # is merely logged to the console. To reproduce the error, we should make - # sure that script's install.js can be run subsequently without unhandled promise rejections. - # Note: the flag `--unahdnled-rejections=strict` will force node to terminate in case - # of UnhandledPromiseRejection. - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node --unhandled-rejections=strict node_modules/playwright/install.js - - echo "${FUNCNAME[0]} success" -} - -function test_skip_browser_download { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - OUTPUT=$(PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --foreground-script ${PLAYWRIGHT_TGZ}) - if [[ "${OUTPUT}" != *"Skipping browsers download because"* ]]; then - echo "missing log message that browsers download is skipped" - exit 1 - fi - - if [[ -d ./node_modules/playwright/.local-browsers ]]; then - echo "local browsers folder should be empty" - exit 1 - fi - - echo "${FUNCNAME[0]} success" -} - -function test_skip_browser_download_inspect_with_custom_executable { - initialize_test "${FUNCNAME[0]}" - copy_test_scripts - - npm install ${PLAYWRIGHT_CORE_TGZ} - OUTPUT=$(PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --foreground-script ${PLAYWRIGHT_TGZ}) - if [[ "${OUTPUT}" != *"Skipping browsers download because"* ]]; then - echo "missing log message that browsers download is skipped" - exit 1 - fi - - if [[ "$(uname)" != "Linux" ]]; then - echo - echo "Skipping test on non-Linux platform" - echo - return - fi - - OUTPUT=$(PWDEBUG=1 node inspector-custom-executable.js) - if [[ "${OUTPUT}" != *"SUCCESS"* ]]; then - echo "missing log message that launch succeeded: ${OUTPUT}" - exit 1 - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - OUTPUT=$(npm install --foreground-script ${PLAYWRIGHT_TGZ}) - if [[ "${OUTPUT}" != *"chromium"* ]]; then - echo "ERROR: should download chromium" - exit 1 - fi - if [[ "${OUTPUT}" != *"firefox"* ]]; then - echo "ERROR: should download firefox" - exit 1 - fi - if [[ "${OUTPUT}" != *"webkit"* ]]; then - echo "ERROR: should download webkit" - exit 1 - fi - copy_test_scripts - - echo "Running sanity.js" - node sanity.js playwright - if [[ ${NODE_VERSION} -ge 14 ]]; then - echo "Running esm.js" - node esm-playwright.mjs - fi - - echo "Running playwright test" - if npx playwright test -c .; then - echo "ERROR: should not be able to run tests with just playwright package" - exit 1 - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_should_work_with_relative_home_path { - initialize_test "${FUNCNAME[0]}" - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_BROWSERS_PATH="" HOME=. npm install ${PLAYWRIGHT_TGZ} - copy_test_scripts - echo "Running sanity.js" - # Firefox does not work with relative HOME. - PLAYWRIGHT_BROWSERS_PATH="" HOME=. node sanity.js playwright chromium webkit - echo "${FUNCNAME[0]} success" -} - -function test_playwright_should_work_with_relative_browsers_path { - initialize_test "${FUNCNAME[0]}" - - # Make sure that browsers path is resolved relative to the `npm install` call location. - mkdir foo - cd foo - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_BROWSERS_PATH="../relative" npm install ${PLAYWRIGHT_TGZ} - cd .. - - copy_test_scripts - echo "Running sanity.js" - PLAYWRIGHT_BROWSERS_PATH="./relative" node sanity.js playwright - echo "${FUNCNAME[0]} success" -} - -function test_playwright_chromium_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - OUTPUT=$(npm install --foreground-script ${PLAYWRIGHT_CHROMIUM_TGZ}) - if [[ "${OUTPUT}" != *"chromium"* ]]; then - echo "ERROR: should download chromium" - exit 1 - fi - if [[ "${OUTPUT}" == *"firefox"* ]]; then - echo "ERROR: should not download firefox" - exit 1 - fi - if [[ "${OUTPUT}" == *"webkit"* ]]; then - echo "ERROR: should not download webkit" - exit 1 - fi - copy_test_scripts - - echo "Running sanity.js" - node sanity.js playwright-chromium - if [[ ${NODE_VERSION} -ge 14 ]]; then - echo "Running esm.js" - node esm-playwright-chromium.mjs - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_webkit_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - OUTPUT=$(npm install --foreground-script ${PLAYWRIGHT_WEBKIT_TGZ}) - if [[ "${OUTPUT}" == *"chromium"* ]]; then - echo "ERROR: should not download chromium" - exit 1 - fi - if [[ "${OUTPUT}" == *"firefox"* ]]; then - echo "ERROR: should not download firefox" - exit 1 - fi - if [[ "${OUTPUT}" != *"webkit"* ]]; then - echo "ERROR: should download webkit" - exit 1 - fi - copy_test_scripts - - echo "Running sanity.js" - node sanity.js playwright-webkit - if [[ ${NODE_VERSION} -ge 14 ]]; then - echo "Running esm.js" - node esm-playwright-webkit.mjs - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_firefox_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - OUTPUT=$(npm install --foreground-script ${PLAYWRIGHT_FIREFOX_TGZ}) - if [[ "${OUTPUT}" == *"chromium"* ]]; then - echo "ERROR: should not download chromium" - exit 1 - fi - if [[ "${OUTPUT}" != *"firefox"* ]]; then - echo "ERROR: should download firefox" - exit 1 - fi - if [[ "${OUTPUT}" == *"webkit"* ]]; then - echo "ERROR: should not download webkit" - exit 1 - fi - copy_test_scripts - - echo "Running sanity.js" - node sanity.js playwright-firefox - if [[ ${NODE_VERSION} -ge 14 ]]; then - echo "Running esm.js" - node esm-playwright-firefox.mjs - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_validate_dependencies { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - npm install ${PLAYWRIGHT_TGZ} - copy_test_scripts - - OUTPUT="$(node validate-dependencies.js)" - if [[ "${OUTPUT}" != *"PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS"* ]]; then - echo "ERROR: validateDependencies was not called" - exit 1 - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_validate_dependencies_skip_executable_path { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - npm install ${PLAYWRIGHT_TGZ} - copy_test_scripts - - OUTPUT="$(node validate-dependencies-skip-executable-path.js)" - if [[ "${OUTPUT}" == *"PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS"* ]]; then - echo "ERROR: validateDependencies was called" - exit 1 - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_electron_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} - npm install electron@9.0 - copy_test_scripts - - echo "Running sanity-electron.js" - node sanity-electron.js - - echo "${FUNCNAME[0]} success" -} - -function test_electron_types { - initialize_test "${FUNCNAME[0]}" - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} - npm install electron@12 - npm install -D typescript@3.8 - npm install -D @types/node@14 - echo "import { Page, _electron, ElectronApplication, Electron } from 'playwright';" > "test.ts" - - echo "Running tsc" - npx -p typescript@3.7.5 tsc "test.ts" - - echo "${FUNCNAME[0]} success" -} - -function test_android_types { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} - npm install -D typescript@3.8 - npm install -D @types/node@14 - echo "import { AndroidDevice, _android, AndroidWebView, Page } from 'playwright';" > "test.ts" - - echo "Running tsc" - npx -p typescript@3.7.5 tsc "test.ts" - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_cli_screenshot_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - npm install ${PLAYWRIGHT_TGZ} - - echo "Running playwright screenshot" - - node_modules/.bin/playwright screenshot about:blank one.png - if [[ ! -f one.png ]]; then - echo 'node_modules/.bin/playwright does not work' - exit 1 - fi - - npx playwright screenshot about:blank two.png - if [[ ! -f two.png ]]; then - echo 'npx playwright does not work' - exit 1 - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_cli_install_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} - - local BROWSERS="$(pwd -P)/browsers" - - echo "Running playwright install chromium" - OUTPUT=$(PLAYWRIGHT_BROWSERS_PATH=${BROWSERS} npx playwright install chromium) - if [[ "${OUTPUT}" != *"chromium"* ]]; then - echo "ERROR: should download chromium" - exit 1 - fi - if [[ "${OUTPUT}" != *"ffmpeg"* ]]; then - echo "ERROR: should download ffmpeg" - exit 1 - fi - if [[ "${OUTPUT}" == *"webkit"* ]]; then - echo "ERROR: should not download webkit" - exit 1 - fi - if [[ "${OUTPUT}" == *"firefox"* ]]; then - echo "ERROR: should not download firefox" - exit 1 - fi - - echo "Running playwright install" - OUTPUT=$(PLAYWRIGHT_BROWSERS_PATH=${BROWSERS} npx playwright install) - if [[ "${OUTPUT}" == *"chromium"* ]]; then - echo "ERROR: should not download chromium" - exit 1 - fi - if [[ "${OUTPUT}" == *"ffmpeg"* ]]; then - echo "ERROR: should not download ffmpeg" - exit 1 - fi - if [[ "${OUTPUT}" != *"webkit"* ]]; then - echo "ERROR: should download webkit" - exit 1 - fi - if [[ "${OUTPUT}" != *"firefox"* ]]; then - echo "ERROR: should download firefox" - exit 1 - fi - - copy_test_scripts - echo "Running sanity.js" - node sanity.js playwright none - PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_cli_codegen_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - npm install ${PLAYWRIGHT_TGZ} - - echo "Running playwright codegen" - OUTPUT=$(PWTEST_CLI_EXIT=1 npx playwright codegen) - if [[ "${OUTPUT}" != *"@playwright/test"* ]]; then - echo "ERROR: missing @playwright/test in the output" - exit 1 - fi - if [[ "${OUTPUT}" != *"{ page }"* ]]; then - echo "ERROR: missing { page } in the output" - exit 1 - fi - - echo "Running playwright codegen --target=javascript" - OUTPUT=$(PWTEST_CLI_EXIT=1 npx playwright codegen --target=javascript) - if [[ "${OUTPUT}" != *"playwright"* ]]; then - echo "ERROR: missing playwright in the output" - exit 1 - fi - if [[ "${OUTPUT}" != *"page.close"* ]]; then - echo "ERROR: missing page.close in the output" - exit 1 - fi - - echo "Running playwright codegen --target=python" - OUTPUT=$(PWTEST_CLI_EXIT=1 npx playwright codegen --target=python) - if [[ "${OUTPUT}" != *"chromium.launch"* ]]; then - echo "ERROR: missing chromium.launch in the output" - exit 1 - fi - if [[ "${OUTPUT}" != *"browser.close"* ]]; then - echo "ERROR: missing browser.close in the output" - exit 1 - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_driver_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} - - echo "Running playwright install" - PLAYWRIGHT_BROWSERS_PATH="0" npx playwright install - - copy_test_scripts - echo "Running driver-client.js" - PLAYWRIGHT_BROWSERS_PATH="0" node driver-client.js - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_test_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - npm install ${PLAYWRIGHT_TEST_TGZ} - copy_test_scripts - - echo "Running playwright test without install" - if npx playwright test -c .; then - echo "ERROR: should not be able to run tests without installing browsers" - exit 1 - fi - - echo "Running playwright install" - PLAYWRIGHT_BROWSERS_PATH="0" npx playwright install - - echo "Running playwright test" - PLAYWRIGHT_JSON_OUTPUT_NAME=report.json PLAYWRIGHT_BROWSERS_PATH="0" npx playwright test -c . --browser=all --reporter=list,json sample.spec.js - - echo "Checking the report" - node ./read-json-report.js ./report.json - - echo "Running sanity.js" - node sanity.js "@playwright/test" - if [[ "${NODE_VERSION}" == *"v14."* ]]; then - echo "Running esm.js" - node esm-playwright-test.mjs - fi - - echo "${FUNCNAME[0]} success" -} - -function test_playwright_test_stacks_should_work { - initialize_test "${FUNCNAME[0]}" - - npm install ${PLAYWRIGHT_CORE_TGZ} - npm install ${PLAYWRIGHT_TEST_TGZ} - PLAYWRIGHT_BROWSERS_PATH="0" npx playwright install chromium - copy_test_scripts - - echo "Running playwright test" - OUTPUT=$(DEBUG=pw:api npx playwright test -c . failing.spec.js 2>&1 || true) - if [[ "${OUTPUT}" != *"expect.toHaveText started"* ]]; then - echo "ERROR: missing 'expect.toHaveText started' in the output" - exit 1 - fi - if [[ "${OUTPUT}" != *"failing.spec.js:5:38"* ]]; then - echo "ERROR: missing 'failing.spec.js:5:38' in the output" - exit 1 - fi - - echo "${FUNCNAME[0]} success" -} - -function test_connect_to_selenium { - node "${SCRIPTS_PATH}/download-chromedriver.js" ${TEST_ROOT} - cd ${SCRIPTS_PATH}/output - PWTEST_CHROMEDRIVER="${TEST_ROOT}/chromedriver" npm run test -- --reporter=list selenium.spec - echo "${FUNCNAME[0]} success" -} - -function initialize_test { - cd ${TEST_ROOT} - local TEST_NAME="./$1" - echo "=====================================================================================" - echo "=====================================================================================" - echo - echo " RUNNING TEST: ${TEST_NAME}" - echo - echo "=====================================================================================" - echo "=====================================================================================" - mkdir ${TEST_NAME} && cd ${TEST_NAME} && npm init -y -} - -# Run all tests -# Script will terminate if there's some error somewhere. -run_tests - -echo -echo "SUCCESS!" diff --git a/installation-tests/run_all_tests.sh b/installation-tests/run_all_tests.sh new file mode 100755 index 0000000000..1a5fcda95d --- /dev/null +++ b/installation-tests/run_all_tests.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -e +set +x + +trap "cd $(pwd -P)" EXIT +cd "$(dirname $0)" + +source ./initialize_test.sh + +setup_env_variables +echo "Building packages..." +build_packages +clean_test_root + +function gh_echo { + if [[ -z "${GITHUB_ACTIONS}" ]]; then + return + fi + echo "$@" +} + +FAILED_TESTS="" + +for i in test_*.sh +do + set +e + cecho "YELLOW" "Running - $i..." + OUTPUT=$(bash $i --multitest --no-build 2>&1) + RV=$? + set -e + if [[ "${RV}" != 0 ]]; then + FAILED_TESTS="${FAILED_TESTS}- bash ${PWD}/${i} --debug\n" + + gh_echo "::group::FAILED - $i" + cecho "RED" "FAILED - $i" + echo "${OUTPUT}" + gh_echo "::endgroup::" + else + gh_echo "::group::PASSED - $i" + cecho "GREEN" "PASSED - $i" + gh_echo "${OUTPUT}" + gh_echo "::endgroup::" + fi +done + +echo +if [[ -n "${FAILED_TESTS}" ]]; then + cecho "RED" "SOME TESTS FAILED! To debug:" + cecho "RED" "${FAILED_TESTS}" + exit 1 +else + cecho "GREEN" "All tests passed!" + exit 0 +fi diff --git a/installation-tests/test_android_types.sh b/installation-tests/test_android_types.sh new file mode 100644 index 0000000000..028c89f496 --- /dev/null +++ b/installation-tests/test_android_types.sh @@ -0,0 +1,11 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} +npm install -D typescript@3.8 +npm install -D @types/node@14 +echo "import { AndroidDevice, _android, AndroidWebView, Page } from 'playwright';" > "test.ts" + +echo "Running tsc" +npx -p typescript@3.7.5 tsc "test.ts" diff --git a/installation-tests/test_connect_to_selenium.sh b/installation-tests/test_connect_to_selenium.sh new file mode 100644 index 0000000000..b2c45a2ca1 --- /dev/null +++ b/installation-tests/test_connect_to_selenium.sh @@ -0,0 +1,6 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +node "${SCRIPTS_PATH}/download-chromedriver.js" ${TEST_ROOT} +cd ${SCRIPTS_PATH}/output +PWTEST_CHROMEDRIVER="${TEST_ROOT}/chromedriver" npm run test -- --reporter=list selenium.spec diff --git a/installation-tests/test_electron_types.sh b/installation-tests/test_electron_types.sh new file mode 100644 index 0000000000..ea9eec0b3d --- /dev/null +++ b/installation-tests/test_electron_types.sh @@ -0,0 +1,13 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} +npm install electron@12 +npm install -D typescript@3.8 +npm install -D @types/node@14 +echo "import { Page, _electron, ElectronApplication, Electron } from 'playwright';" > "test.ts" + +echo "Running tsc" +npx -p typescript@3.7.5 tsc "test.ts" + diff --git a/installation-tests/test_playwright_chromium_should_work.sh b/installation-tests/test_playwright_chromium_should_work.sh new file mode 100644 index 0000000000..3a7491ee3b --- /dev/null +++ b/installation-tests/test_playwright_chromium_should_work.sh @@ -0,0 +1,25 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +OUTPUT=$(npm install --foreground-script ${PLAYWRIGHT_CHROMIUM_TGZ}) +if [[ "${OUTPUT}" != *"chromium"* ]]; then + echo "ERROR: should download chromium" + exit 1 +fi +if [[ "${OUTPUT}" == *"firefox"* ]]; then + echo "ERROR: should not download firefox" + exit 1 +fi +if [[ "${OUTPUT}" == *"webkit"* ]]; then + echo "ERROR: should not download webkit" + exit 1 +fi +copy_test_scripts + +echo "Running sanity.js" +node sanity.js playwright-chromium +if [[ ${NODE_VERSION} -ge 14 ]]; then + echo "Running esm.js" + node esm-playwright-chromium.mjs +fi diff --git a/installation-tests/test_playwright_cli_codegen_should_work.sh b/installation-tests/test_playwright_cli_codegen_should_work.sh new file mode 100644 index 0000000000..735cf2b35f --- /dev/null +++ b/installation-tests/test_playwright_cli_codegen_should_work.sh @@ -0,0 +1,38 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +npm install ${PLAYWRIGHT_TGZ} + +echo "Running playwright codegen" +OUTPUT=$(PWTEST_CLI_EXIT=1 npx playwright codegen) +if [[ "${OUTPUT}" != *"@playwright/test"* ]]; then + echo "ERROR: missing @playwright/test in the output" + exit 1 +fi +if [[ "${OUTPUT}" != *"{ page }"* ]]; then + echo "ERROR: missing { page } in the output" + exit 1 +fi + +echo "Running playwright codegen --target=javascript" +OUTPUT=$(PWTEST_CLI_EXIT=1 npx playwright codegen --target=javascript) +if [[ "${OUTPUT}" != *"playwright"* ]]; then + echo "ERROR: missing playwright in the output" + exit 1 +fi +if [[ "${OUTPUT}" != *"page.close"* ]]; then + echo "ERROR: missing page.close in the output" + exit 1 +fi + +echo "Running playwright codegen --target=python" +OUTPUT=$(PWTEST_CLI_EXIT=1 npx playwright codegen --target=python) +if [[ "${OUTPUT}" != *"chromium.launch"* ]]; then + echo "ERROR: missing chromium.launch in the output" + exit 1 +fi +if [[ "${OUTPUT}" != *"browser.close"* ]]; then + echo "ERROR: missing browser.close in the output" + exit 1 +fi diff --git a/installation-tests/test_playwright_cli_install_should_work.sh b/installation-tests/test_playwright_cli_install_should_work.sh new file mode 100644 index 0000000000..a2d232571a --- /dev/null +++ b/installation-tests/test_playwright_cli_install_should_work.sh @@ -0,0 +1,51 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} + +BROWSERS="$(pwd -P)/browsers" + +echo "Running playwright install chromium" +OUTPUT=$(PLAYWRIGHT_BROWSERS_PATH=${BROWSERS} npx playwright install chromium) +if [[ "${OUTPUT}" != *"chromium"* ]]; then + echo "ERROR: should download chromium" + exit 1 +fi +if [[ "${OUTPUT}" != *"ffmpeg"* ]]; then + echo "ERROR: should download ffmpeg" + exit 1 +fi +if [[ "${OUTPUT}" == *"webkit"* ]]; then + echo "ERROR: should not download webkit" + exit 1 +fi +if [[ "${OUTPUT}" == *"firefox"* ]]; then + echo "ERROR: should not download firefox" + exit 1 +fi + +echo "Running playwright install" +OUTPUT=$(PLAYWRIGHT_BROWSERS_PATH=${BROWSERS} npx playwright install) +if [[ "${OUTPUT}" == *"chromium"* ]]; then + echo "ERROR: should not download chromium" + exit 1 +fi +if [[ "${OUTPUT}" == *"ffmpeg"* ]]; then + echo "ERROR: should not download ffmpeg" + exit 1 +fi +if [[ "${OUTPUT}" != *"webkit"* ]]; then + echo "ERROR: should download webkit" + exit 1 +fi +if [[ "${OUTPUT}" != *"firefox"* ]]; then + echo "ERROR: should download firefox" + exit 1 +fi + +copy_test_scripts +echo "Running sanity.js" +node sanity.js playwright none +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright + diff --git a/installation-tests/test_playwright_cli_screenshot_should_work.sh b/installation-tests/test_playwright_cli_screenshot_should_work.sh new file mode 100644 index 0000000000..4f0beaf3ad --- /dev/null +++ b/installation-tests/test_playwright_cli_screenshot_should_work.sh @@ -0,0 +1,19 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +npm install ${PLAYWRIGHT_TGZ} + +echo "Running playwright screenshot" + +node_modules/.bin/playwright screenshot about:blank one.png +if [[ ! -f one.png ]]; then + echo 'node_modules/.bin/playwright does not work' + exit 1 +fi + +npx playwright screenshot about:blank two.png +if [[ ! -f two.png ]]; then + echo 'npx playwright does not work' + exit 1 +fi diff --git a/installation-tests/test_playwright_driver_should_work.sh b/installation-tests/test_playwright_driver_should_work.sh new file mode 100644 index 0000000000..007b8b3f6e --- /dev/null +++ b/installation-tests/test_playwright_driver_should_work.sh @@ -0,0 +1,13 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} + +echo "Running playwright install" +PLAYWRIGHT_BROWSERS_PATH="0" npx playwright install + +copy_test_scripts +echo "Running driver-client.js" +PLAYWRIGHT_BROWSERS_PATH="0" node driver-client.js + diff --git a/installation-tests/test_playwright_electron_should_work.sh b/installation-tests/test_playwright_electron_should_work.sh new file mode 100644 index 0000000000..804378614e --- /dev/null +++ b/installation-tests/test_playwright_electron_should_work.sh @@ -0,0 +1,10 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} +npm install electron@9.0 +copy_test_scripts + +echo "Running sanity-electron.js" +node sanity-electron.js diff --git a/installation-tests/test_playwright_firefox_should_work.sh b/installation-tests/test_playwright_firefox_should_work.sh new file mode 100644 index 0000000000..4938970cb4 --- /dev/null +++ b/installation-tests/test_playwright_firefox_should_work.sh @@ -0,0 +1,25 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +OUTPUT=$(npm install --foreground-script ${PLAYWRIGHT_FIREFOX_TGZ}) +if [[ "${OUTPUT}" == *"chromium"* ]]; then + echo "ERROR: should not download chromium" + exit 1 +fi +if [[ "${OUTPUT}" != *"firefox"* ]]; then + echo "ERROR: should download firefox" + exit 1 +fi +if [[ "${OUTPUT}" == *"webkit"* ]]; then + echo "ERROR: should not download webkit" + exit 1 +fi +copy_test_scripts + +echo "Running sanity.js" +node sanity.js playwright-firefox +if [[ ${NODE_VERSION} -ge 14 ]]; then + echo "Running esm.js" + node esm-playwright-firefox.mjs +fi diff --git a/installation-tests/test_playwright_global_installation.sh b/installation-tests/test_playwright_global_installation.sh new file mode 100644 index 0000000000..334ed544ae --- /dev/null +++ b/installation-tests/test_playwright_global_installation.sh @@ -0,0 +1,16 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +BROWSERS="$(pwd -P)/browsers" +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_TGZ} +if [[ ! -d "${BROWSERS}" ]]; then + echo "Directory for shared browsers was not created!" + exit 1 +fi + +copy_test_scripts + +echo "Running sanity.js" +node sanity.js playwright none +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright diff --git a/installation-tests/test_playwright_global_installation_cross_package.sh b/installation-tests/test_playwright_global_installation_cross_package.sh new file mode 100644 index 0000000000..1ef388f60a --- /dev/null +++ b/installation-tests/test_playwright_global_installation_cross_package.sh @@ -0,0 +1,22 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_FIREFOX_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_WEBKIT_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_CHROMIUM_TGZ} + +BROWSERS="$(pwd -P)/browsers" +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_TGZ} +if [[ ! -d "${BROWSERS}" ]]; then + echo "Directory for shared browsers was not created!" + exit 1 +fi + +copy_test_scripts + +echo "Running sanity.js" +# Every package should be able to launch. +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright-chromium all +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright-firefox all +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright-webkit all diff --git a/installation-tests/test_playwright_global_installation_subsequent_installs.sh b/installation-tests/test_playwright_global_installation_subsequent_installs.sh new file mode 100644 index 0000000000..b98c45c6d7 --- /dev/null +++ b/installation-tests/test_playwright_global_installation_subsequent_installs.sh @@ -0,0 +1,17 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +# @see https://github.com/microsoft/playwright/issues/1651 + +BROWSERS="$(pwd -P)/browsers" + +mkdir install-1 && pushd install-1 && npm init -y +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_TGZ} +# Note: the `npm install` would not actually crash, the error +# is merely logged to the console. To reproduce the error, we should make +# sure that script's install.js can be run subsequently without unhandled promise rejections. +# Note: the flag `--unahdnled-rejections=strict` will force node to terminate in case +# of UnhandledPromiseRejection. +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node --unhandled-rejections=strict node_modules/playwright/install.js + diff --git a/installation-tests/test_playwright_should_work.sh b/installation-tests/test_playwright_should_work.sh new file mode 100644 index 0000000000..ddf4998af0 --- /dev/null +++ b/installation-tests/test_playwright_should_work.sh @@ -0,0 +1,32 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +OUTPUT=$(npm install --foreground-script ${PLAYWRIGHT_TGZ}) +if [[ "${OUTPUT}" != *"chromium"* ]]; then + echo "ERROR: should download chromium" + exit 1 +fi +if [[ "${OUTPUT}" != *"firefox"* ]]; then + echo "ERROR: should download firefox" + exit 1 +fi +if [[ "${OUTPUT}" != *"webkit"* ]]; then + echo "ERROR: should download webkit" + exit 1 +fi +copy_test_scripts + +echo "Running sanity.js" +node sanity.js playwright +if [[ ${NODE_VERSION} -ge 14 ]]; then + echo "Running esm.js" + node esm-playwright.mjs +fi + +echo "Running playwright test" +if npx playwright test -c .; then + echo "ERROR: should not be able to run tests with just playwright package" + exit 1 +fi + diff --git a/installation-tests/test_playwright_should_work_with_relative_browsers_path.sh b/installation-tests/test_playwright_should_work_with_relative_browsers_path.sh new file mode 100644 index 0000000000..220c22ce47 --- /dev/null +++ b/installation-tests/test_playwright_should_work_with_relative_browsers_path.sh @@ -0,0 +1,14 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +# Make sure that browsers path is resolved relative to the `npm install` call location. +mkdir foo +cd foo +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_BROWSERS_PATH="../relative" npm install ${PLAYWRIGHT_TGZ} +cd .. + +copy_test_scripts +echo "Running sanity.js" +PLAYWRIGHT_BROWSERS_PATH="./relative" node sanity.js playwright + diff --git a/installation-tests/test_playwright_should_work_with_relative_home_path.sh b/installation-tests/test_playwright_should_work_with_relative_home_path.sh new file mode 100644 index 0000000000..a751ec2fd7 --- /dev/null +++ b/installation-tests/test_playwright_should_work_with_relative_home_path.sh @@ -0,0 +1,9 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_BROWSERS_PATH="" HOME=. npm install ${PLAYWRIGHT_TGZ} +copy_test_scripts +echo "Running sanity.js" +# Firefox does not work with relative HOME. +PLAYWRIGHT_BROWSERS_PATH="" HOME=. node sanity.js playwright chromium webkit diff --git a/installation-tests/test_playwright_test_should_work.sh b/installation-tests/test_playwright_test_should_work.sh new file mode 100644 index 0000000000..7fce2db234 --- /dev/null +++ b/installation-tests/test_playwright_test_should_work.sh @@ -0,0 +1,29 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +npm install ${PLAYWRIGHT_TEST_TGZ} +copy_test_scripts + +echo "Running playwright test without install" +if npx playwright test -c .; then + echo "ERROR: should not be able to run tests without installing browsers" + exit 1 +fi + +echo "Running playwright install" +PLAYWRIGHT_BROWSERS_PATH="0" npx playwright install + +echo "Running playwright test" +PLAYWRIGHT_JSON_OUTPUT_NAME=report.json PLAYWRIGHT_BROWSERS_PATH="0" npx playwright test -c . --browser=all --reporter=list,json sample.spec.js + +echo "Checking the report" +node ./read-json-report.js ./report.json + +echo "Running sanity.js" +node sanity.js "@playwright/test" +if [[ "${NODE_VERSION}" == *"v14."* ]]; then + echo "Running esm.js" + node esm-playwright-test.mjs +fi + diff --git a/installation-tests/test_playwright_test_stacks_should_work.sh b/installation-tests/test_playwright_test_stacks_should_work.sh new file mode 100644 index 0000000000..728223abee --- /dev/null +++ b/installation-tests/test_playwright_test_stacks_should_work.sh @@ -0,0 +1,18 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +npm install ${PLAYWRIGHT_TEST_TGZ} +PLAYWRIGHT_BROWSERS_PATH="0" npx playwright install chromium +copy_test_scripts + +echo "Running playwright test" +OUTPUT=$(DEBUG=pw:api npx playwright test -c . failing.spec.js 2>&1 || true) +if [[ "${OUTPUT}" != *"expect.toHaveText started"* ]]; then + echo "ERROR: missing 'expect.toHaveText started' in the output" + exit 1 +fi +if [[ "${OUTPUT}" != *"failing.spec.js:5:38"* ]]; then + echo "ERROR: missing 'failing.spec.js:5:38' in the output" + exit 1 +fi diff --git a/installation-tests/test_playwright_validate_dependencies.sh b/installation-tests/test_playwright_validate_dependencies.sh new file mode 100644 index 0000000000..0e3bd0ef62 --- /dev/null +++ b/installation-tests/test_playwright_validate_dependencies.sh @@ -0,0 +1,13 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +npm install ${PLAYWRIGHT_TGZ} +copy_test_scripts + +OUTPUT="$(node validate-dependencies.js)" +if [[ "${OUTPUT}" != *"PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS"* ]]; then + echo "ERROR: validateDependencies was not called" + exit 1 +fi + diff --git a/installation-tests/test_playwright_validate_dependencies_skip_executable_path.sh b/installation-tests/test_playwright_validate_dependencies_skip_executable_path.sh new file mode 100644 index 0000000000..c8d2659e89 --- /dev/null +++ b/installation-tests/test_playwright_validate_dependencies_skip_executable_path.sh @@ -0,0 +1,13 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +npm install ${PLAYWRIGHT_TGZ} +copy_test_scripts + +OUTPUT="$(node validate-dependencies-skip-executable-path.js)" +if [[ "${OUTPUT}" == *"PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS"* ]]; then + echo "ERROR: validateDependencies was called" + exit 1 +fi + diff --git a/installation-tests/test_playwright_webkit_should_work.sh b/installation-tests/test_playwright_webkit_should_work.sh new file mode 100644 index 0000000000..4aee65c200 --- /dev/null +++ b/installation-tests/test_playwright_webkit_should_work.sh @@ -0,0 +1,25 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +OUTPUT=$(npm install --foreground-script ${PLAYWRIGHT_WEBKIT_TGZ}) +if [[ "${OUTPUT}" == *"chromium"* ]]; then + echo "ERROR: should not download chromium" + exit 1 +fi +if [[ "${OUTPUT}" == *"firefox"* ]]; then + echo "ERROR: should not download firefox" + exit 1 +fi +if [[ "${OUTPUT}" != *"webkit"* ]]; then + echo "ERROR: should download webkit" + exit 1 +fi +copy_test_scripts + +echo "Running sanity.js" +node sanity.js playwright-webkit +if [[ ${NODE_VERSION} -ge 14 ]]; then + echo "Running esm.js" + node esm-playwright-webkit.mjs +fi diff --git a/installation-tests/test_screencast.sh b/installation-tests/test_screencast.sh new file mode 100644 index 0000000000..f1d7c5fb9a --- /dev/null +++ b/installation-tests/test_screencast.sh @@ -0,0 +1,18 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +copy_test_scripts + +BROWSERS="$(pwd -P)/browsers" +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_TGZ} +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_FIREFOX_TGZ} +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_WEBKIT_TGZ} +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_CHROMIUM_TGZ} + +echo "Running screencast.js" +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node screencast.js playwright +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node screencast.js playwright-chromium +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node screencast.js playwright-webkit +PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node screencast.js playwright-firefox + diff --git a/installation-tests/test_skip_browser_download.sh b/installation-tests/test_skip_browser_download.sh new file mode 100644 index 0000000000..57f247ab0d --- /dev/null +++ b/installation-tests/test_skip_browser_download.sh @@ -0,0 +1,15 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +npm install ${PLAYWRIGHT_CORE_TGZ} +OUTPUT=$(PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --foreground-script ${PLAYWRIGHT_TGZ}) +if [[ "${OUTPUT}" != *"Skipping browsers download because"* ]]; then + echo "missing log message that browsers download is skipped" + exit 1 +fi + +if [[ -d ./node_modules/playwright/.local-browsers ]]; then + echo "local browsers folder should be empty" + exit 1 +fi + diff --git a/installation-tests/test_skip_browser_download_inspect_with_custom_executable.sh b/installation-tests/test_skip_browser_download_inspect_with_custom_executable.sh new file mode 100644 index 0000000000..51962b6ff0 --- /dev/null +++ b/installation-tests/test_skip_browser_download_inspect_with_custom_executable.sh @@ -0,0 +1,25 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +copy_test_scripts + +npm install ${PLAYWRIGHT_CORE_TGZ} +OUTPUT=$(PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --foreground-script ${PLAYWRIGHT_TGZ}) +if [[ "${OUTPUT}" != *"Skipping browsers download because"* ]]; then + echo "missing log message that browsers download is skipped" + exit 1 +fi + +if [[ "$(uname)" != "Linux" ]]; then + echo + echo "Skipping test on non-Linux platform" + echo + return +fi + +OUTPUT=$(PWDEBUG=1 node inspector-custom-executable.js) +if [[ "${OUTPUT}" != *"SUCCESS"* ]]; then + echo "missing log message that launch succeeded: ${OUTPUT}" + exit 1 +fi + diff --git a/installation-tests/test_typescript_types.sh b/installation-tests/test_typescript_types.sh new file mode 100644 index 0000000000..2cf4fd4668 --- /dev/null +++ b/installation-tests/test_typescript_types.sh @@ -0,0 +1,30 @@ +#!/bin/bash +source ./initialize_test.sh && initialize_test "$@" + +copy_test_scripts +# @types/node@14.18.9 is the last version which is compatibel with typescript@3.7.5. +# After @types/node@14.18.9 URLSearchParams from @types/node conflicts with typescript's +# shipped types and it results in a type error / build failure. +npm install -D @types/node@14.18.9 + +# install all packages. +npm install ${PLAYWRIGHT_CORE_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_FIREFOX_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_WEBKIT_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_CHROMIUM_TGZ} +PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TEST_TGZ} + +# typecheck all packages. +for PKG_NAME in "playwright" \ + "playwright-core" \ + "playwright-firefox" \ + "playwright-chromium" \ + "playwright-webkit" +do + echo "Checking types of ${PKG_NAME}" + echo "import { Page } from '${PKG_NAME}';" > "${PKG_NAME}.ts" && npx -p typescript@3.7.5 tsc "${PKG_NAME}.ts" +done; + +echo "Checking types of @playwright/test" +echo npx -p typescript@3.7.5 tsc "playwright-test-types.ts"