chore: improve installation tests (#12092)

* chore: improve installation tests

- all helper scripts and files are moved to `fixture-scripts`
  subfolder.
- `./run_all_tests.sh` now shows a counter to estimate progress
- function `copy_test_scripts` is no longer needed; all fixture
  scripts are automatically copied to test folder

Co-authored-by: Max Schmitt <max@schmitt.mx>

Co-authored-by: Max Schmitt <max@schmitt.mx>

installation-tests/README.md

@@ -1,4 +1,18 @@
 # Installation Tests
 
-File `installation-tests.sh` tests that installation flow for all
+These tests check end-to-end installation and operation of Playwright.
-Playwright packages works as expected.
+Each test is set with a separate folder that contains all scripts from
+`fixture-scripts` folder and dummy package.json.
+
+To create a new test, create a new file that starts with `test_*.sh`
+with the following header:
+    ```bash
+    #!/bin/bash
+    source ./initialize_test.sh && initialize_test "$@" # initialize test
+    ```
+
+To run all tests:
+
+```bash
+./run_all_tests.sh
+```

installation-tests/fixture-scripts/download-chromedriver.js

@@ -1,4 +1,4 @@
-const playwright = require('playwright');
+const playwright = require('playwright-core');
 const { execSync } = require('child_process');
 const path = require('path');
 

installation-tests/initialize_test.sh

@@ -3,6 +3,8 @@
 # break script execution if some command returns non-zero exit code
 set -e
 
+TEST_FRAMEWORK_RUN_ROOT="/tmp/playwright-installation-tests"
+
 function build_packages() {
   local PACKAGE_BUILDER="../../utils/pack_package.js"
   rm -rf ./output
@@ -26,7 +28,7 @@ function cecho(){
   printf "${!1}${2} ${NC}\n"
 }
 
-function complete_test {
+function report_test_result {
   set +x
   if [[ $? == 0 ]]; then
     echo
@@ -37,7 +39,7 @@ function complete_test {
     cecho "RED" "<<<<<<<<<<<<"
     cecho "RED" "  Test '${TEST_FILE}' FAILED"
     cecho "RED" "  To debug locally, run:"
-    cecho "RED" "       bash ${TEST_PATH} --debug"
+    cecho "RED" "       bash ${TEST_FILE}"
     cecho "RED" "<<<<<<<<<<<<"
     echo
   fi
@@ -46,8 +48,6 @@ function complete_test {
 
 function setup_env_variables() {
   # Package paths.
-  SCRIPTS_PATH="$(pwd -P)"
-  TEST_ROOT="/tmp/playwright-installation-tests"
   NODE_VERSION=$(node -e "console.log(process.version.slice(1).split('.')[0])")
 
   PLAYWRIGHT_CORE_TGZ="${PWD}/output/playwright-core.tgz"
@@ -56,15 +56,16 @@ function setup_env_variables() {
   PLAYWRIGHT_WEBKIT_TGZ="${PWD}/output/playwright-webkit.tgz"
   PLAYWRIGHT_FIREFOX_TGZ="${PWD}/output/playwright-firefox.tgz"
   PLAYWRIGHT_TEST_TGZ="${PWD}/output/playwright-test.tgz"
+  PLAYWRIGHT_CHECKOUT="${PWD}/.."
 }
 
 function clean_test_root() {
-  rm -rf "${TEST_ROOT}"
+  rm -rf "${TEST_FRAMEWORK_RUN_ROOT}"
-  mkdir -p "${TEST_ROOT}"
+  mkdir -p "${TEST_FRAMEWORK_RUN_ROOT}"
 }
 
 function initialize_test {
-  trap "complete_test;cd $(pwd -P)" EXIT
+  trap "report_test_result;cd $(pwd -P)" EXIT
   cd "$(dirname $0)"
 
   # cleanup environment
@@ -72,6 +73,7 @@ function initialize_test {
   unset PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD
   export PLAYWRIGHT_BROWSERS_PATH=0
 
+  local SCRIPTS_PATH="$(pwd -P)"
   setup_env_variables
 
   if [[ "$1" != "--no-build" && "$2" != "--no-build" ]]; then
@@ -91,40 +93,18 @@ function initialize_test {
   if [[ "$1" != "--do-not-clean-test-root" && "$2" != "--do-not-clean-test-root" ]]; then
     clean_test_root
   fi
-  cd ${TEST_ROOT}
+  cd ${TEST_FRAMEWORK_RUN_ROOT}
   TEST_FILE=$(basename $0)
   TEST_NAME=$(basename ${0%%.sh})
-  TEST_PATH="${PWD}/${TEST_FILE}"
 
   cecho "YELLOW" ">>>>>>>>>>>>"
   cecho "YELLOW" "  Running test - '${TEST_FILE}'"
   cecho "YELLOW" ">>>>>>>>>>>>"
   mkdir ${TEST_NAME} && cd ${TEST_NAME} && npm init -y 1>/dev/null 2>/dev/null
 
-  # enable bash lines logging if --debug is passed
+  cp "${SCRIPTS_PATH}/fixture-scripts/"* .
-  if [[ $1 == "--debug" || $2 == "--debug" ]]; then
+
-    set -x
+  # Enable bash lines logging.
-  fi
+  set -x
-}
-
-function copy_test_scripts {
-  cp "${SCRIPTS_PATH}/inspector-custom-executable.js" .
-  cp "${SCRIPTS_PATH}/sanity.js" .
-  cp "${SCRIPTS_PATH}/screencast.js" .
-  cp "${SCRIPTS_PATH}/validate-dependencies.js" .
-  cp "${SCRIPTS_PATH}/validate-dependencies-skip-executable-path.js" .
-  cp "${SCRIPTS_PATH}/esm.mjs" .
-  cp "${SCRIPTS_PATH}/esm-playwright.mjs" .
-  cp "${SCRIPTS_PATH}/esm-playwright-chromium.mjs" .
-  cp "${SCRIPTS_PATH}/esm-playwright-firefox.mjs" .
-  cp "${SCRIPTS_PATH}/esm-playwright-webkit.mjs" .
-  cp "${SCRIPTS_PATH}/esm-playwright-test.mjs" .
-  cp "${SCRIPTS_PATH}/sanity-electron.js" .
-  cp "${SCRIPTS_PATH}/electron-app.js" .
-  cp "${SCRIPTS_PATH}/driver-client.js" .
-  cp "${SCRIPTS_PATH}/sample.spec.js" .
-  cp "${SCRIPTS_PATH}/failing.spec.js" .
-  cp "${SCRIPTS_PATH}/read-json-report.js" .
-  cp "${SCRIPTS_PATH}/playwright-test-types.ts" .
 }
 

installation-tests/run_all_tests.sh

@@ -2,7 +2,19 @@
 set -e
 set +x
 
-trap "cd $(pwd -P)" EXIT
+function report_results() {
+  echo
+  if [[ -n "${FAILED_TESTS}" ]]; then
+    cecho "RED" "SOME TESTS FAILED! To debug:"
+    cecho "RED" "${FAILED_TESTS}"
+    exit 1
+  else
+    cecho "GREEN" "All tests passed!"
+    exit 0
+  fi
+}
+
+trap "report_results; cd $(pwd -P)" EXIT
 cd "$(dirname $0)"
 
 source ./initialize_test.sh
@@ -21,15 +33,18 @@ function gh_echo {
 
 FAILED_TESTS=""
 
+TOTAL=$(ls -1 test_*.sh | wc -l | tr -d ' ')
+COUNTER=1
 for i in test_*.sh
 do
   set +e
-  cecho "YELLOW" "Running - $i..."
+  cecho "YELLOW" "Running ${COUNTER}/${TOTAL} - $i..."
+  COUNTER=$(( COUNTER + 1 ))
   OUTPUT=$(bash $i --multitest --no-build 2>&1)
   RV=$?
   set -e
   if [[ "${RV}" != 0 ]]; then
-    FAILED_TESTS="${FAILED_TESTS}- bash ${PWD}/${i} --debug\n"
+    FAILED_TESTS="${FAILED_TESTS}- ${i}\n"
 
     gh_echo "::group::FAILED - $i"
     cecho "RED" "FAILED - $i"
@@ -43,12 +58,3 @@ do
   fi
 done
 
-echo
-if [[ -n "${FAILED_TESTS}" ]]; then
-  cecho "RED" "SOME TESTS FAILED! To debug:"
-  cecho "RED" "${FAILED_TESTS}"
-  exit 1
-else
-  cecho "GREEN" "All tests passed!"
-  exit 0
-fi

installation-tests/test_connect_to_selenium.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 source ./initialize_test.sh && initialize_test "$@"
 
-node "${SCRIPTS_PATH}/download-chromedriver.js" ${TEST_ROOT}
+npm install ${PLAYWRIGHT_CORE_TGZ}
-cd ${SCRIPTS_PATH}/output
+node "./download-chromedriver.js" "${PWD}"
-PWTEST_CHROMEDRIVER="${TEST_ROOT}/chromedriver" npm run test -- --reporter=list selenium.spec
+export PWTEST_CHROMEDRIVER="${PWD}/chromedriver"
+cd "${PLAYWRIGHT_CHECKOUT}"
+npm run test -- --reporter=list selenium.spec

installation-tests/test_playwright_chromium_should_work.sh

@@ -15,7 +15,6 @@ if [[ "${OUTPUT}" == *"webkit"* ]]; then
   echo "ERROR: should not download webkit"
   exit 1
 fi
-copy_test_scripts
 
 echo "Running sanity.js"
 node sanity.js playwright-chromium

installation-tests/test_playwright_cli_install_should_work.sh

@@ -44,7 +44,6 @@ if [[ "${OUTPUT}" != *"firefox"* ]]; then
   exit 1
 fi
 
-copy_test_scripts
 echo "Running sanity.js"
 node sanity.js playwright none
 PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright

installation-tests/test_playwright_driver_should_work.sh

@@ -7,7 +7,6 @@ PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ}
 echo "Running playwright install"
 PLAYWRIGHT_BROWSERS_PATH="0" npx playwright install
 
-copy_test_scripts
 echo "Running driver-client.js"
 PLAYWRIGHT_BROWSERS_PATH="0" node driver-client.js
 

installation-tests/test_playwright_electron_should_work.sh

@@ -4,7 +4,6 @@ source ./initialize_test.sh && initialize_test "$@"
 npm install ${PLAYWRIGHT_CORE_TGZ}
 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TGZ}
 npm install electron@9.0
-copy_test_scripts
 
 echo "Running sanity-electron.js"
 node sanity-electron.js

installation-tests/test_playwright_firefox_should_work.sh

@@ -15,7 +15,6 @@ if [[ "${OUTPUT}" == *"webkit"* ]]; then
   echo "ERROR: should not download webkit"
   exit 1
 fi
-copy_test_scripts
 
 echo "Running sanity.js"
 node sanity.js playwright-firefox

installation-tests/test_playwright_global_installation.sh

@@ -9,8 +9,6 @@ if [[ ! -d "${BROWSERS}" ]]; then
   exit 1
 fi
 
-copy_test_scripts
-
 echo "Running sanity.js"
 node sanity.js playwright none
 PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright

installation-tests/test_playwright_global_installation_cross_package.sh

@@ -13,8 +13,6 @@ if [[ ! -d "${BROWSERS}" ]]; then
   exit 1
 fi
 
-copy_test_scripts
-
 echo "Running sanity.js"
 # Every package should be able to launch.
 PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" node sanity.js playwright-chromium all

installation-tests/test_playwright_should_work.sh

@@ -15,7 +15,6 @@ if [[ "${OUTPUT}" != *"webkit"* ]]; then
   echo "ERROR: should download webkit"
   exit 1
 fi
-copy_test_scripts
 
 echo "Running sanity.js"
 node sanity.js playwright

installation-tests/test_playwright_should_work_with_relative_browsers_path.sh

@@ -8,7 +8,6 @@ npm install ${PLAYWRIGHT_CORE_TGZ}
 PLAYWRIGHT_BROWSERS_PATH="../relative" npm install ${PLAYWRIGHT_TGZ}
 cd ..
 
-copy_test_scripts
 echo "Running sanity.js"
 PLAYWRIGHT_BROWSERS_PATH="./relative" node sanity.js playwright
 

installation-tests/test_playwright_should_work_with_relative_home_path.sh

@@ -3,7 +3,6 @@ source ./initialize_test.sh && initialize_test "$@"
 
 npm install ${PLAYWRIGHT_CORE_TGZ}
 PLAYWRIGHT_BROWSERS_PATH="" HOME=. npm install ${PLAYWRIGHT_TGZ}
-copy_test_scripts
 echo "Running sanity.js"
 # Firefox does not work with relative HOME.
 PLAYWRIGHT_BROWSERS_PATH="" HOME=. node sanity.js playwright chromium webkit

installation-tests/test_playwright_test_should_work.sh

@@ -3,7 +3,6 @@ source ./initialize_test.sh && initialize_test "$@"
 
 npm install ${PLAYWRIGHT_CORE_TGZ}
 npm install ${PLAYWRIGHT_TEST_TGZ}
-copy_test_scripts
 
 echo "Running playwright test without install"
 if npx playwright test -c .; then

installation-tests/test_playwright_test_stacks_should_work.sh

@@ -4,7 +4,6 @@ source ./initialize_test.sh && initialize_test "$@"
 npm install ${PLAYWRIGHT_CORE_TGZ}
 npm install ${PLAYWRIGHT_TEST_TGZ}
 PLAYWRIGHT_BROWSERS_PATH="0" npx playwright install chromium
-copy_test_scripts
 
 echo "Running playwright test"
 OUTPUT=$(DEBUG=pw:api npx playwright test -c . failing.spec.js 2>&1 || true)

installation-tests/test_playwright_validate_dependencies.sh

@@ -3,7 +3,6 @@ source ./initialize_test.sh && initialize_test "$@"
 
 npm install ${PLAYWRIGHT_CORE_TGZ}
 npm install ${PLAYWRIGHT_TGZ}
-copy_test_scripts
 
 OUTPUT="$(node validate-dependencies.js)"
 if [[ "${OUTPUT}" != *"PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS"* ]]; then

installation-tests/test_playwright_validate_dependencies_skip_executable_path.sh

@@ -3,7 +3,6 @@ source ./initialize_test.sh && initialize_test "$@"
 
 npm install ${PLAYWRIGHT_CORE_TGZ}
 npm install ${PLAYWRIGHT_TGZ}
-copy_test_scripts
 
 OUTPUT="$(node validate-dependencies-skip-executable-path.js)"
 if [[ "${OUTPUT}" == *"PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS"* ]]; then

installation-tests/test_playwright_webkit_should_work.sh

@@ -15,7 +15,6 @@ if [[ "${OUTPUT}" != *"webkit"* ]]; then
   echo "ERROR: should download webkit"
   exit 1
 fi
-copy_test_scripts
 
 echo "Running sanity.js"
 node sanity.js playwright-webkit

installation-tests/test_screencast.sh

@@ -1,8 +1,6 @@
 #!/bin/bash
 source ./initialize_test.sh && initialize_test "$@"
 
-copy_test_scripts
-
 BROWSERS="$(pwd -P)/browsers"
 npm install ${PLAYWRIGHT_CORE_TGZ}
 PLAYWRIGHT_BROWSERS_PATH="${BROWSERS}" npm install ${PLAYWRIGHT_TGZ}

installation-tests/test_skip_browser_download_inspect_with_custom_executable.sh

@@ -1,8 +1,6 @@
 #!/bin/bash
 source ./initialize_test.sh && initialize_test "$@"
 
-copy_test_scripts
-
 npm install ${PLAYWRIGHT_CORE_TGZ}
 OUTPUT=$(PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --foreground-script ${PLAYWRIGHT_TGZ})
 if [[ "${OUTPUT}" != *"Skipping browsers download because"* ]]; then

installation-tests/test_typescript_types.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 source ./initialize_test.sh && initialize_test "$@"
 
-copy_test_scripts
 # @types/node@14.18.9 is the last version which is compatibel with typescript@3.7.5.
 # After @types/node@14.18.9 URLSearchParams from @types/node conflicts with typescript's
 # shipped types and it results in a type error / build failure.