playwright/.github/workflows/tests.yml

name: tests

on:
  push:
    branches:
      - master
      - release-*
  pull_request:
    branches:
      - master
      - release-*

env:
  # Force terminal colors. @see https://www.npmjs.com/package/colors
  FORCE_COLOR: 1
  FLAKINESS_CONNECTION_STRING: ${{ secrets.FLAKINESS_CONNECTION_STRING }}

jobs:
  test_linux:
    name: ${{ matrix.os }} (${{ matrix.browser }})
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
        os: [ubuntu-18.04, ubuntu-20.04]
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
    - run: npm run build
    - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --tag=${{ matrix.browser }}"
    - run: node tests/config/checkCoverage.js ${{ matrix.browser }}
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: always()
      with:
        name: ${{ matrix.browser }}-${{ matrix.os }}-test-results
        path: test-results

  test_mac:
    name: ${{ matrix.os }} (${{ matrix.browser }})
    strategy:
      fail-fast: false
      matrix:
        os: [macos-10.15, macos-11.0]
        browser: [chromium, firefox, webkit]
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
      env:
        DEBUG=extract-zip
    - run: npm run build
    - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
    - run: npm run test -- --tag=${{ matrix.browser }}
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: ${{ matrix.browser }}-${{ matrix.os }}-test-results
        path: test-results

  test_win:
    name: "Windows"
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
    runs-on: windows-latest
    steps:
    - name: Install Media Pack
      shell: powershell
      run: Install-WindowsFeature Server-Media-Foundation
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
      env:
        DEBUG=extract-zip
    - run: npm run build
    - run: node lib/cli/cli install-deps
    - run: npm run test -- --tag=${{ matrix.browser }}
      shell: bash
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: ${{ matrix.browser }}-win-test-results
        path: test-results

  test-package-installations:
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        node_version:
        - "^12.0.0"
        - "^14.1.0"  # pre 14.1, zip extraction was broken (https://github.com/microsoft/playwright/issues/1988)
    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: ${{ matrix.node_version }}
    - run: npm ci
      env:
        DEBUG=extract-zip
    - run: npm run build
    - run: node lib/cli/cli install-deps
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash packages/installation-tests/installation-tests.sh

  headful_linux:
    name: "Headful Linux"
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
    - run: npm run build
    - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --tag=${{ matrix.browser }}"
      if: ${{ always() }}
      env:
        HEADFUL: 1
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: headful-${{ matrix.browser }}-linux-test-results
        path: test-results

  transport_linux:
    name: "Transport"
    strategy:
      fail-fast: false
      matrix:
        mode: [driver, service]
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
    - run: npm run build
    - run: node lib/cli/cli install-deps chromium
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest"
      env:
        PWTEST_MODE: "${{ matrix.mode }}"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: mode-${{ matrix.mode }}-linux-test-results
        path: test-results

  video_linux:
    name: "Video Linux"
    strategy:
      fail-fast: false
      matrix:
        browser: [chromium, firefox, webkit]
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
    - run: npm run build
    - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run test -- --tag=${{ matrix.browser }}"
      env:
        PWTEST_VIDEO: 1
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: video-${{ matrix.browser }}-linux-test-results
        path: test-results

  test_android:
    name: Android Emulator (shard ${{ matrix.shard }})
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2]
    runs-on: macos-10.15
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 14
    - run: npm ci
    - run: npm run build
    - run: node lib/cli/cli install-deps
    - name: Create Android Emulator
      run: utils/avd_recreate.sh
    - name: Start Android Emulator
      run: utils/avd_start.sh
    - name: Run tests
      run: npm run atest -- --shard=${{ matrix.shard }}/2
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: android-test-results
        path: test-results

  chrome_stable_linux:
    name: "Chrome Stable (Linux)"
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - name: Install Chrome Stable
      run: sudo apt install google-chrome-stable
    - run: npm ci
      env:
        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
    - run: npm run build
    - run: node lib/cli/cli install-deps chromium
    # This only created problems, should we move ffmpeg back into npm?
    - run: node lib/cli/cli install ffmpeg
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ctest"
      env:
        PWTEST_CHANNEL: "chrome"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: chrome-stable-linux-test-results
        path: test-results

  chrome_stable_win:
    name: "Chrome Stable (Win)"
    runs-on: windows-latest
    steps:
    - name: Install Media Pack
      shell: powershell
      run: Install-WindowsFeature Server-Media-Foundation
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
      env:
        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
    - run: npm run build
    # This only created problems, should we move ffmpeg back into npm?
    - run: node lib/cli/cli install ffmpeg
    - run: npm run ctest
      shell: bash
      env:
        PWTEST_CHANNEL: "chrome"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: chrome-stable-win-test-results
        path: test-results

  chrome_stable_mac:
    name: "Chrome Stable (Mac)"
    runs-on: macos-10.15
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
      env:
        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
    - run: npm run build
    # This only created problems, should we move ffmpeg back into npm?
    - run: node lib/cli/cli install ffmpeg
    - run: npm run ctest
      env:
        PWTEST_CHANNEL: "chrome"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: chrome-stable-mac-test-results
        path: test-results

  firefox_stable_linux:
    name: "Firefox Stable (Linux)"
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
      env:
        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
    - run: npm run build
    - run: node lib/cli/cli install-deps firefox
    - run: node lib/cli/cli install ffmpeg firefox-stable chromium
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run ftest"
      env:
        PWTEST_CHANNEL: "firefox-stable"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: firefox-stable-linux-test-results
        path: test-results

  firefox_stable_win:
    name: "Firefox Stable (Win)"
    runs-on: windows-latest
    steps:
    - name: Install Media Pack
      shell: powershell
      run: Install-WindowsFeature Server-Media-Foundation
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
      env:
        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
    - run: npm run build
    - run: node lib/cli/cli install ffmpeg firefox-stable chromium
    - run: npm run ftest
      shell: bash
      env:
        PWTEST_CHANNEL: "firefox-stable"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: firefox-stable-win-test-results
        path: test-results

  firefox_stable_mac:
    name: "Firefox Stable (Mac)"
    runs-on: macos-10.15
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
      env:
        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
    - run: npm run build
    - run: node lib/cli/cli install ffmpeg firefox-stable chromium
    - run: npm run ftest
      env:
        PWTEST_CHANNEL: "firefox-stable"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: firefox-stable-mac-test-results
        path: test-results

  edge_stable_win:
    name: "Edge Stable (Win)"
    runs-on: windows-latest
    steps:
    - name: Install Media Pack
      shell: powershell
      run: Install-WindowsFeature Server-Media-Foundation
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
      env:
        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
    - run: npm run build
    # This only created problems, should we move ffmpeg back into npm?
    - run: node lib/cli/cli install ffmpeg
    - run: npm run ctest
      shell: bash
      env:
        PWTEST_CHANNEL: "msedge"
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: edge-stable-win-test-results
        path: test-results

  test_electron:
    name: "Electron Linux"
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-node@v2
      with:
        node-version: 12
    - run: npm ci
    - run: npm run build
    - run: node lib/cli/cli install-deps chromium
    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
    # Wrap `npm run` in a subshell to redirect STDERR to file.
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "npm run etest"
    - run: node tests/config/checkCoverage.js electron
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
    - uses: actions/upload-artifact@v1
      if: ${{ always() }}
      with:
        name: electron-linux-test-results
        path: test-results