playwright/.github/workflows/tests_secondary.yml
Yury Semikhatsky 972ad89af2
feat(devops): publish merged reports to pull requests (#23419)
* Moved report merging and publishing logic into create_test_report.yml
shared between all workflows
* Merged reports are now published for try jobs on pull requests too. In
order to achieve that the logic had to be extracted into a separate
workflow triggered by
[workflow_run](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_run),
this way it can access secrets even if the original workflow was not
able to.
* The blob report data flow is different depending on whether the
workflow is triggered by a pull request or a push:
- For `pull_request` the workflow doesn't have access to the secrets it
uploads the blob report to the GitHub
artifact storage. Later on the merge workflow uploads that blob report
to Azure blob storage.
- Workflows triggered by `push` event can read secrets. They upload blob
report directly to Azure blob storage
and the merge workflow downloads the report from there rather than from
GitHub artifacts.
2023-06-01 08:40:43 -07:00

659 lines
19 KiB
YAML

name: "tests 2"
on:
push:
branches:
- main
- release-*
pull_request:
paths-ignore:
- 'browser_patches/**'
- 'docs/**'
types: [ labeled ]
branches:
- main
- release-*
env:
# Force terminal colors. @see https://www.npmjs.com/package/colors
FORCE_COLOR: 1
FLAKINESS_CONNECTION_STRING: ${{ secrets.FLAKINESS_CONNECTION_STRING }}
ELECTRON_SKIP_BINARY_DOWNLOAD: 1
jobs:
test_linux:
name: ${{ matrix.os }} (${{ matrix.browser }})
strategy:
fail-fast: false
matrix:
browser: [chromium, firefox, webkit]
os: [ubuntu-20.04, ubuntu-22.04]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
DEBUG: pw:install
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps ${{ matrix.browser }} chromium
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}
- run: node tests/config/checkCoverage.js ${{ matrix.browser }}
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
test_mac:
name: ${{ matrix.os }} (${{ matrix.browser }})
strategy:
fail-fast: false
matrix:
os: [macos-12, macos-13]
browser: [chromium, firefox, webkit]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
DEBUG: pw:install
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps ${{ matrix.browser }} chromium
- run: npm run test -- --project=${{ matrix.browser }}
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
test_win:
name: "Windows"
strategy:
fail-fast: false
matrix:
browser: [chromium, firefox, webkit]
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
DEBUG: pw:install
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps ${{ matrix.browser }} chromium
- run: npm run test -- --project=${{ matrix.browser }}
shell: bash
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
test-package-installations-other-node-versions:
name: "Installation Test ${{ matrix.os }} (${{ matrix.node_version }})"
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
node_version: 18
- os: ubuntu-latest
node_version: 20
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node_version }}
- run: npm ci
env:
DEBUG: pw:install
- run: npm run build
- run: npx playwright install-deps
- run: npm run itest
if: matrix.os != 'ubuntu-latest'
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run itest
if: matrix.os == 'ubuntu-latest'
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
headed_tests:
name: "headed ${{ matrix.browser }} (${{ matrix.os }})"
strategy:
fail-fast: false
matrix:
browser: [chromium, firefox, webkit]
os: [ubuntu-20.04, ubuntu-22.04, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
DEBUG: pw:install
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps ${{ matrix.browser }} chromium
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} --headed
if: always() && startsWith(matrix.os, 'ubuntu-')
- run: npm run test -- --project=${{ matrix.browser }} --headed
if: always() && !startsWith(matrix.os, 'ubuntu-')
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
transport_linux:
name: "Transport"
strategy:
fail-fast: false
matrix:
mode: [driver, service]
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
DEBUG: pw:install
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps chromium
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
env:
PWTEST_MODE: ${{ matrix.mode }}
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
tracing_linux:
name: Tracing ${{ matrix.browser }} ${{ matrix.channel }}
strategy:
fail-fast: false
matrix:
include:
- browser: chromium
- browser: firefox
- browser: webkit
- browser: chromium
channel: chromium-tip-of-tree
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
DEBUG: pw:install
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps ${{ matrix.browser }} chromium ${{ matrix.channel }}
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}
env:
PWTEST_TRACE: 1
PWTEST_CHANNEL: ${{ matrix.channel }}
PWTEST_BLOB_REPORT: 1
PWTEST_BLOB_SUFFIX: ${{ (matrix.channel && format('-{0}', matrix.channel)) || '' }}
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
- name: Upload blob report to Azure
if: always() && github.event_name == 'push'
run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.sha }}' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
- name: Upload blob report to GitHub
uses: actions/upload-artifact@v3
if: always() && github.event_name == 'pull_request'
with:
name: blob-report-${{ github.run_attempt }}
path: blob-report
retention-days: 30
chrome_stable_linux:
name: "Chrome Stable (Linux)"
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps chrome
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
env:
PWTEST_CHANNEL: chrome
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
chrome_stable_win:
name: "Chrome Stable (Win)"
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps chrome
- run: npm run ctest
shell: bash
env:
PWTEST_CHANNEL: chrome
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
chrome_stable_mac:
name: "Chrome Stable (Mac)"
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps chrome
- run: npm run ctest
env:
PWTEST_CHANNEL: chrome
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
chromium_tot:
name: Chromium TOT ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, macos-12, windows-latest]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps chromium-tip-of-tree
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
if: matrix.os == 'ubuntu-20.04'
env:
PWTEST_CHANNEL: chromium-tip-of-tree
- run: npm run ctest
if: matrix.os != 'ubuntu-20.04'
env:
PWTEST_CHANNEL: chromium-tip-of-tree
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
chromium_tot_headed:
name: Chromium TOT headed ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps chromium-tip-of-tree
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest -- --headed
if: matrix.os == 'ubuntu-latest'
env:
PWTEST_CHANNEL: chromium-tip-of-tree
- run: npm run ctest -- --headed
if: matrix.os != 'ubuntu-latest'
env:
PWTEST_CHANNEL: chromium-tip-of-tree
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
firefox_beta_linux:
name: "Firefox Beta (Linux)"
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps firefox-beta chromium
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ftest
env:
PWTEST_CHANNEL: firefox-beta
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
firefox_beta_win:
name: "Firefox Beta (Win)"
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps firefox-beta chromium
- run: npm run ftest
shell: bash
env:
PWTEST_CHANNEL: firefox-beta
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
firefox_beta_mac:
name: "Firefox Beta (Mac)"
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps firefox-beta chromium
- run: npm run ftest
env:
PWTEST_CHANNEL: firefox-beta
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
edge_stable_mac:
name: "Edge Stable (Mac)"
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps msedge
- run: npm run ctest
env:
PWTEST_CHANNEL: msedge
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
edge_stable_win:
name: "Edge Stable (Win)"
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps msedge
- run: npm run ctest
shell: bash
env:
PWTEST_CHANNEL: msedge
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
edge_stable_linux:
name: "Edge Stable (Linux)"
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps msedge
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
env:
PWTEST_CHANNEL: msedge
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
edge_beta_mac:
name: "Edge Beta (Mac)"
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps msedge-beta
- run: npm run ctest
env:
PWTEST_CHANNEL: msedge-beta
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
edge_beta_win:
name: "Edge Beta (Win)"
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps msedge-beta
- run: npm run ctest
shell: bash
env:
PWTEST_CHANNEL: msedge-beta
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
edge_beta_linux:
name: "Edge Beta (Linux)"
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps msedge-beta
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
env:
PWTEST_CHANNEL: msedge-beta
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
edge_dev_mac:
name: "Edge Dev (Mac)"
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps msedge-dev
- run: npm run ctest
env:
PWTEST_CHANNEL: msedge-dev
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
edge_dev_win:
name: "Edge Dev (Win)"
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps msedge-dev
- run: npm run ctest
shell: bash
env:
PWTEST_CHANNEL: msedge-dev
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
edge_dev_linux:
name: "Edge Dev (Linux)"
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps msedge-dev
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
env:
PWTEST_CHANNEL: msedge-dev
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
chrome_beta_linux:
name: "Chrome Beta (Linux)"
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps chrome-beta
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
env:
PWTEST_CHANNEL: chrome-beta
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
chrome_beta_win:
name: "Chrome Beta (Win)"
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps chrome-beta
- run: npm run ctest
shell: bash
env:
PWTEST_CHANNEL: chrome-beta
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
chrome_beta_mac:
name: "Chrome Beta (Mac)"
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
- run: npm run build
- run: npx playwright install --with-deps chrome-beta
- run: npm run ctest
env:
PWTEST_CHANNEL: chrome-beta
- run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
if: always()
shell: bash
build-playwright-driver:
name: "build-playwright-driver"
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16
- run: npm ci
- run: npm run build
- run: npx playwright install-deps
- run: utils/build/build-playwright-driver.sh