devops: migrate to download/upload-artifact@v4 (#28850)

`actions/upload-artifact@v4` comes with the following [breaking change](https://github.com/actions/upload-artifact?tab=readme-ov-file#breaking-changes): "Due to how Artifacts are created in this new version, it is no longer possible to upload to the same named Artifact multiple times. You must either split the uploads into multiple Artifacts with different names, or only upload once. Otherwise you will encounter an error." Due to that we cannot copy multiple blob report folders into the same artifact name and rely on the action to merge them. Instead, as suggested by their migration guide, we upload each blob report into a uniquely named artifact with prefix `blob-report-` and then download all of them into same directory. This version change also affects how we store pull_request_number.txt into an artifact. Previously we relied on the fact that uploading artifact with the same name would silently override existing one, but now it's an error. To overcome that, we upload PR number file into uniquely named artifacts `pull-request-*` and later extract them into same location with `unzip -n` which will never override existing file, so we end up with single `pull_request_number.txt`. Reference #28800
2024-01-04 09:13:55 -08:00 · 2024-01-04 09:13:55 -08:00 · 6c2c777cae
parent edf369ea3c
commit 6c2c777cae
5 changed files with 151 additions and 91 deletions
--- a/.github/actions/download-artifact/action.yml
+++ b/.github/actions/download-artifact/action.yml
@ -1,8 +1,8 @@
-name: 'Download blob report'
-description: 'Download blob report from GitHub artifacts'
+name: 'Download artifacts'
+description: 'Download artifacts from GitHub'
 inputs:
-  name:
-    description: 'Name of the artifact to download'
+  namePrefix:
+    description: 'Name prefix of the artifacts to download'
    required: true
    type: string
    default: 'blob-report'
@ -10,11 +10,14 @@ inputs:
    description: 'Directory with downloaded artifacts'
    required: true
    type: string
-    default: 'blob-report'
+    default: '.'
 runs:
  using: "composite"
  steps:
-    - name: Download blob report
+    - name: Create temp downloads dir
+      shell: bash
+      run: mkdir -p '${{ inputs.path }}/artifacts'
+    - name: Download artifacts
      uses: actions/github-script@v6
      with:
        script: |
@ -25,16 +28,19 @@ runs:
            run_id: context.payload.workflow_run.id
          });
          console.log('total = ', data.total_count);
-          const name = '${{ inputs.name }}';
-          const report = data.artifacts.filter(a => a.name === name)[0];
-          const result = await github.rest.actions.downloadArtifact({
-            ...context.repo,
-            artifact_id: report.id,
-            archive_format: 'zip'
-          });
-          console.log('download result', result);
+          const artifacts = data.artifacts.filter(a => a.name.startsWith('${{ inputs.namePrefix }}'));
          const fs = require('fs');
-          fs.writeFileSync(`${name}.zip`, Buffer.from(result.data));
-    - name: Unzip blob report
+          for (const artifact of artifacts) {
+            const result = await github.rest.actions.downloadArtifact({
+              ...context.repo,
+              artifact_id: artifact.id,
+              archive_format: 'zip'
+            });
+            console.log('downloaded artifact', result);
+            fs.writeFileSync(`${{ inputs.path }}/artifacts/${artifact.name}.zip`, Buffer.from(result.data));
+          }
+    - name: Unzip artifacts
      shell: bash
-      run: unzip ${{ inputs.name }}.zip -d ${{ inputs.path }}
+      run: |
+        unzip -n '${{ inputs.path }}/artifacts/*.zip' -d ${{ inputs.path }}
+        rm -rf '${{ inputs.path }}/artifacts'
--- a/.github/actions/upload-blob-report/action.yml
+++ b/.github/actions/upload-blob-report/action.yml
@ -5,15 +5,20 @@ inputs:
    description: 'Directory containing blob report'
    required: true
    type: string
-    default: 'blob-report'
+    default: 'test-results/blob-report'
+  job_name:
+    description: 'Unique job name'
+    required: true
+    type: string
+    default: ''
 runs:
  using: "composite"
  steps:
    - name: Upload blob report to GitHub
      if: always() && github.event_name == 'pull_request'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
      with:
-        name: all-blob-reports
+        name: blob-report-${{ inputs.job_name }}
        path: ${{ inputs.report_dir }}
        retention-days: 7
    - name: Write triggering pull request number in a file
@ -22,7 +27,7 @@ runs:
      run: echo '${{ github.event.number }}' > pull_request_number.txt;
    - name: Upload artifact with the pull request number
      if: always() && github.event_name == 'pull_request'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
      with:
-        name: pull-request
-        path: pull_request_number.txt
+        name: pull-request-${{ inputs.job_name }}
+        path: pull_request_number.txt
--- a/.github/workflows/create_test_report.yml
+++ b/.github/workflows/create_test_report.yml
@ -26,8 +26,8 @@ jobs:
    - name: Download blob report artifact
      uses: ./.github/actions/download-artifact
      with:
-        name: all-blob-reports
-        path: all-blob-reports
+        namePrefix: 'blob-report'
+        path: 'all-blob-reports'

    - name: Merge reports
      run: |
@ -49,8 +49,8 @@ jobs:
    - name: Read pull request number
      uses: ./.github/actions/download-artifact
      with:
-        name: 'pull-request'
-        path: './'
+        namePrefix: 'pull-request'
+        path: '.'

    - name: Comment on PR
      uses: actions/github-script@v6
--- a/.github/workflows/tests_primary.yml
+++ b/.github/workflows/tests_primary.yml
@ -42,6 +42,8 @@ jobs:
            node-version: 20
            browser: chromium
    runs-on: ${{ matrix.os }}
+    env:
+      PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -54,8 +56,6 @@ jobs:
    - run: npm run build
    - run: npx playwright install --with-deps ${{ matrix.browser }} chromium
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}
-      env:
-        PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}"
    - run: node tests/config/checkCoverage.js ${{ matrix.browser }}
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
@ -65,6 +65,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  test_linux_chromium_tot:
    name: ${{ matrix.os }} (chromium tip-of-tree)
@ -73,6 +74,8 @@ jobs:
      matrix:
        os: [ubuntu-20.04]
    runs-on: ${{ matrix.os }}
+    env:
+      PWTEST_BOT_NAME: "${{ matrix.os }}-chromium-tip-of-tree"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -96,6 +99,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  test_test_runner:
    name: Test Runner
@ -104,21 +108,28 @@ jobs:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        node-version: [18]
-        shard: [1/2, 2/2]
+        shardIndex: [1, 2]
+        shardTotal: [2]
        include:
          - os: ubuntu-latest
            node-version: 16
-            shard: 1/2
+            shardIndex: 1
+            shardTotal: 2
          - os: ubuntu-latest
            node-version: 16
-            shard: 2/2
+            shardIndex: 2
+            shardTotal: 2
          - os: ubuntu-latest
            node-version: 20
-            shard: 1/2
+            shardIndex: 1
+            shardTotal: 2
          - os: ubuntu-latest
            node-version: 20
-            shard: 2/2
+            shardIndex: 2
+            shardTotal: 2
    runs-on: ${{ matrix.os }}
+    env:
+      PWTEST_BOT_NAME: "${{ matrix.os }}-node${{ matrix.node-version }}-${{ matrix.shardIndex }}"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -129,13 +140,9 @@ jobs:
        DEBUG: pw:install
    - run: npm run build
    - run: npx playwright install --with-deps
-    - run: npm run ttest -- --shard ${{ matrix.shard }}
-      env:
-        PWTEST_BOT_NAME: "${{ matrix.os }}-node${{ matrix.node-version }}-${{ matrix.shard }}"
+    - run: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
      if: matrix.os != 'ubuntu-latest'
-    - run: xvfb-run npm run ttest --  --shard ${{ matrix.shard }}
-      env:
-        PWTEST_BOT_NAME: "${{ matrix.os }}-node${{ matrix.node-version }}-${{ matrix.shard }}"
+    - run: xvfb-run npm run ttest --  --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
      if: matrix.os == 'ubuntu-latest'
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
@ -145,6 +152,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  test_web_components:
    name: Web Components
@ -168,6 +176,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: packages/html-reporter/blob-report
+        job_name: "web-components-html-reporter"

    - run: npm run test-web
      if: always()
@ -178,10 +187,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: packages/web/blob-report
+        job_name: "web-components-web"

  test_vscode_extension:
    name: VSCode Extension
    runs-on: ubuntu-latest
+    env:
+      PWTEST_BOT_NAME: "vscode-extension"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -205,18 +217,16 @@ jobs:
      working-directory: ./playwright-vscode
    - name: Run extension tests
      run: npm run test -- --workers=1
-      env:
-        PWTEST_BOT_NAME: "vscode-extension"
      working-directory: ./playwright-vscode
    - name: Upload blob report
      if: always()
      uses: ./.github/actions/upload-blob-report
      with:
-        report_dir: ./playwright-vscode/blob-report
+        report_dir: playwright-vscode/blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  test_package_installations:
    name: "Installation Test ${{ matrix.os }}"
-    runs-on: ${{ matrix.os  }}
    strategy:
      fail-fast: false
      matrix:
@ -224,7 +234,10 @@ jobs:
        - ubuntu-latest
        - macos-latest
        - windows-latest
+    runs-on: ${{ matrix.os  }}
    timeout-minutes: 30
+    env:
+      PWTEST_BOT_NAME: "package-installations-${{ matrix.os }}"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -239,12 +252,8 @@ jobs:
    - run: npm install -g pnpm@8
    - run: npm run itest
      if: matrix.os != 'ubuntu-latest'
-      env:
-        PWTEST_BOT_NAME: "package-installations-${{ matrix.os }}"
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run itest
      if: matrix.os == 'ubuntu-latest'
-      env:
-        PWTEST_BOT_NAME: "package-installations-${{ matrix.os }}"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -253,3 +262,4 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}
--- a/.github/workflows/tests_secondary.yml
+++ b/.github/workflows/tests_secondary.yml
@ -29,6 +29,8 @@ jobs:
        browser: [chromium, firefox, webkit]
        os: [ubuntu-20.04]
    runs-on: ${{ matrix.os }}
+    env:
+      PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -41,8 +43,6 @@ jobs:
    - run: npm run build
    - run: npx playwright install --with-deps ${{ matrix.browser }} chromium
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}
-      env:
-        PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}"
    - run: node tests/config/checkCoverage.js ${{ matrix.browser }}
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
@ -52,6 +52,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  test_mac:
    name: ${{ matrix.os }} (${{ matrix.browser }})
@ -61,6 +62,8 @@ jobs:
        os: [macos-12, macos-13]
        browser: [chromium, firefox, webkit]
    runs-on: ${{ matrix.os }}
+    env:
+      PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -73,8 +76,6 @@ jobs:
    - run: npm run build
    - run: npx playwright install --with-deps ${{ matrix.browser }} chromium
    - run: npm run test -- --project=${{ matrix.browser }}
-      env:
-        PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -83,6 +84,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  test_win:
    name: "Windows"
@ -91,6 +93,8 @@ jobs:
      matrix:
        browser: [chromium, firefox, webkit]
    runs-on: windows-latest
+    env:
+      PWTEST_BOT_NAME: "${{ matrix.browser }}-windows-latest"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -105,13 +109,9 @@ jobs:
    - run: npm run test -- --project=${{ matrix.browser }} --workers=1
      if: matrix.browser == 'firefox'
      shell: bash
-      env:
-        PWTEST_BOT_NAME: "${{ matrix.browser }}-windows-latest"
    - run: npm run test -- --project=${{ matrix.browser }}
      if: matrix.browser != 'firefox'
      shell: bash
-      env:
-        PWTEST_BOT_NAME: "${{ matrix.browser }}-windows-latest"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -120,6 +120,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  test-package-installations-other-node-versions:
    name: "Installation Test ${{ matrix.os }} (${{ matrix.node_version }})"
@ -161,6 +162,8 @@ jobs:
        browser: [chromium, firefox, webkit]
        os: [ubuntu-20.04, ubuntu-22.04, macos-latest, windows-latest]
    runs-on: ${{ matrix.os }}
+    env:
+      PWTEST_BOT_NAME: "${{ matrix.browser }}-headed-${{ matrix.os }}"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -174,12 +177,8 @@ jobs:
    - run: npx playwright install --with-deps ${{ matrix.browser }} chromium
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} --headed
      if: always() && startsWith(matrix.os, 'ubuntu-')
-      env:
-        PWTEST_BOT_NAME: "${{ matrix.browser }}-headed-${{ matrix.os }}"
    - run: npm run test -- --project=${{ matrix.browser }} --headed
      if: always() && !startsWith(matrix.os, 'ubuntu-')
-      env:
-        PWTEST_BOT_NAME: "${{ matrix.browser }}-headed-${{ matrix.os }}"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -188,6 +187,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  transport_linux:
    name: "Transport"
@ -196,6 +196,8 @@ jobs:
      matrix:
        mode: [driver, service]
    runs-on: ubuntu-20.04
+    env:
+      PWTEST_BOT_NAME: "${{ matrix.mode }}"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -210,7 +212,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
      env:
        PWTEST_MODE: ${{ matrix.mode }}
-        PWTEST_BOT_NAME: "${{ matrix.mode }}"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -219,6 +220,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  tracing_linux:
    name: Tracing ${{ matrix.browser }} ${{ matrix.channel }}
@ -232,6 +234,8 @@ jobs:
          - browser: chromium
            channel: chromium-tip-of-tree
    runs-on: ubuntu-20.04
+    env:
+      PWTEST_BOT_NAME: "tracing-${{ matrix.channel || matrix.browser }}"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -247,7 +251,6 @@ jobs:
      env:
        PWTEST_TRACE: 1
        PWTEST_CHANNEL: ${{ matrix.channel }}
-        PWTEST_BOT_NAME: "tracing-${{ matrix.channel || matrix.browser }}"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -256,10 +259,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  chrome_stable_linux:
    name: "Chrome Stable (Linux)"
    runs-on: ubuntu-20.04
+    env:
+      PWTEST_BOT_NAME: "chrome-stable-linux"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -273,7 +279,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
      env:
        PWTEST_CHANNEL: chrome
-        PWTEST_BOT_NAME: "chrome-stable-linux"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -282,10 +287,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  chrome_stable_win:
    name: "Chrome Stable (Win)"
    runs-on: windows-latest
+    env:
+      PWTEST_BOT_NAME: "chrome-stable-windows"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -300,7 +308,6 @@ jobs:
      shell: bash
      env:
        PWTEST_CHANNEL: chrome
-        PWTEST_BOT_NAME: "chrome-stable-windows"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -309,10 +316,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  chrome_stable_mac:
    name: "Chrome Stable (Mac)"
    runs-on: macos-latest
+    env:
+      PWTEST_BOT_NAME: "chrome-stable-mac"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -326,7 +336,6 @@ jobs:
    - run: npm run ctest
      env:
        PWTEST_CHANNEL: chrome
-        PWTEST_BOT_NAME: "chrome-stable-mac"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -335,10 +344,14 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  chromium_tot:
    name: Chromium TOT ${{ matrix.os }}
    runs-on: ${{ matrix.os  }}
+    env:
+      PWTEST_CHANNEL: chromium-tip-of-tree
+      PWTEST_BOT_NAME: "tip-of-tree-${{ matrix.os }}"
    strategy:
      fail-fast: false
      matrix:
@ -355,14 +368,8 @@ jobs:
    - run: npx playwright install --with-deps chromium-tip-of-tree
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
      if: matrix.os == 'ubuntu-20.04'
-      env:
-        PWTEST_CHANNEL: chromium-tip-of-tree
-        PWTEST_BOT_NAME: "tip-of-tree-${{ matrix.os }}"
    - run: npm run ctest
      if: matrix.os != 'ubuntu-20.04'
-      env:
-        PWTEST_CHANNEL: chromium-tip-of-tree
-        PWTEST_BOT_NAME: "tip-of-tree-${{ matrix.os }}"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -371,10 +378,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  chromium_tot_headed:
    name: Chromium TOT headed ${{ matrix.os }}
    runs-on: ${{ matrix.os  }}
+    env:
+      PWTEST_BOT_NAME: "tip-of-tree-headed-${{ matrix.os }}"
    strategy:
      fail-fast: false
      matrix:
@ -391,14 +401,10 @@ jobs:
    - run: npx playwright install --with-deps chromium-tip-of-tree
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest -- --headed
      if: matrix.os == 'ubuntu-latest'
-      env:
-        PWTEST_CHANNEL: chromium-tip-of-tree
-        PWTEST_BOT_NAME: "tip-of-tree-headed-${{ matrix.os }}"
    - run: npm run ctest -- --headed
      if: matrix.os != 'ubuntu-latest'
      env:
        PWTEST_CHANNEL: chromium-tip-of-tree
-        PWTEST_BOT_NAME: "tip-of-tree-headed-${{ matrix.os }}"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -407,10 +413,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  firefox_beta_linux:
    name: "Firefox Beta (Linux)"
    runs-on: ubuntu-20.04
+    env:
+      PWTEST_BOT_NAME: "firefox-beta-linux"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -424,7 +433,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ftest
      env:
        PWTEST_CHANNEL: firefox-beta
-        PWTEST_BOT_NAME: "firefox-beta-linux"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -433,10 +441,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  firefox_beta_win:
    name: "Firefox Beta (Win)"
    runs-on: windows-latest
+    env:
+      PWTEST_BOT_NAME: "firefox-beta-windows"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -451,7 +462,6 @@ jobs:
      shell: bash
      env:
        PWTEST_CHANNEL: firefox-beta
-        PWTEST_BOT_NAME: "firefox-beta-windows"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -460,10 +470,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  firefox_beta_mac:
    name: "Firefox Beta (Mac)"
    runs-on: macos-latest
+    env:
+      PWTEST_BOT_NAME: "firefox-beta-mac"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -477,7 +490,6 @@ jobs:
    - run: npm run ftest
      env:
        PWTEST_CHANNEL: firefox-beta
-        PWTEST_BOT_NAME: "firefox-beta-mac"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -486,10 +498,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  edge_stable_mac:
    name: "Edge Stable (Mac)"
    runs-on: macos-latest
+    env:
+      PWTEST_BOT_NAME: "edge-stable-mac"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -503,7 +518,6 @@ jobs:
    - run: npm run ctest
      env:
        PWTEST_CHANNEL: msedge
-        PWTEST_BOT_NAME: "edge-stable-mac"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -512,10 +526,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  edge_stable_win:
    name: "Edge Stable (Win)"
    runs-on: windows-latest
+    env:
+      PWTEST_BOT_NAME: "edge-stable-windows"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -530,7 +547,6 @@ jobs:
      shell: bash
      env:
        PWTEST_CHANNEL: msedge
-        PWTEST_BOT_NAME: "edge-stable-windows"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -539,10 +555,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  edge_stable_linux:
    name: "Edge Stable (Linux)"
    runs-on: ubuntu-20.04
+    env:
+      PWTEST_BOT_NAME: "edge-stable-linux"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -556,7 +575,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
      env:
        PWTEST_CHANNEL: msedge
-        PWTEST_BOT_NAME: "edge-stable-linux"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -565,10 +583,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  edge_beta_mac:
    name: "Edge Beta (Mac)"
    runs-on: macos-latest
+    env:
+      PWTEST_BOT_NAME: "edge-beta-mac"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -582,7 +603,6 @@ jobs:
    - run: npm run ctest
      env:
        PWTEST_CHANNEL: msedge-beta
-        PWTEST_BOT_NAME: "edge-beta-mac"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -591,10 +611,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  edge_beta_win:
    name: "Edge Beta (Win)"
    runs-on: windows-latest
+    env:
+      PWTEST_BOT_NAME: "edge-beta-windows"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -609,7 +632,6 @@ jobs:
      shell: bash
      env:
        PWTEST_CHANNEL: msedge-beta
-        PWTEST_BOT_NAME: "edge-beta-windows"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -618,10 +640,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  edge_beta_linux:
    name: "Edge Beta (Linux)"
    runs-on: ubuntu-20.04
+    env:
+      PWTEST_BOT_NAME: "edge-beta-linux"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -635,7 +660,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
      env:
        PWTEST_CHANNEL: msedge-beta
-        PWTEST_BOT_NAME: "edge-beta-linux"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -644,10 +668,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  edge_dev_mac:
    name: "Edge Dev (Mac)"
    runs-on: macos-latest
+    env:
+      PWTEST_BOT_NAME: "edge-dev-mac"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -661,7 +688,6 @@ jobs:
    - run: npm run ctest
      env:
        PWTEST_CHANNEL: msedge-dev
-        PWTEST_BOT_NAME: "edge-dev-mac"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -670,10 +696,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  edge_dev_win:
    name: "Edge Dev (Win)"
    runs-on: windows-latest
+    env:
+      PWTEST_BOT_NAME: "edge-dev-windows"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -688,7 +717,6 @@ jobs:
      shell: bash
      env:
        PWTEST_CHANNEL: msedge-dev
-        PWTEST_BOT_NAME: "edge-dev-windows"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -697,10 +725,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  edge_dev_linux:
    name: "Edge Dev (Linux)"
    runs-on: ubuntu-20.04
+    env:
+      PWTEST_BOT_NAME: "edge-dev-linux"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -714,7 +745,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
      env:
        PWTEST_CHANNEL: msedge-dev
-        PWTEST_BOT_NAME: "edge-dev-linux"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -723,10 +753,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  chrome_beta_linux:
    name: "Chrome Beta (Linux)"
    runs-on: ubuntu-20.04
+    env:
+      PWTEST_BOT_NAME: "chrome-beta-linux"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -740,7 +773,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run ctest
      env:
        PWTEST_CHANNEL: chrome-beta
-        PWTEST_BOT_NAME: "chrome-beta-linux"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -749,10 +781,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  chrome_beta_win:
    name: "Chrome Beta (Win)"
    runs-on: windows-latest
+    env:
+      PWTEST_BOT_NAME: "chrome-beta-windows"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -767,7 +802,6 @@ jobs:
      shell: bash
      env:
        PWTEST_CHANNEL: chrome-beta
-        PWTEST_BOT_NAME: "chrome-beta-windows"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -776,10 +810,13 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  chrome_beta_mac:
    name: "Chrome Beta (Mac)"
    runs-on: macos-latest
+    env:
+      PWTEST_BOT_NAME: "chrome-beta-mac"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -793,7 +830,6 @@ jobs:
    - run: npm run ctest
      env:
        PWTEST_CHANNEL: chrome-beta
-        PWTEST_BOT_NAME: "chrome-beta-mac"
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
      shell: bash
@ -802,6 +838,7 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}

  build-playwright-driver:
    name: "build-playwright-driver"
@ -819,6 +856,8 @@ jobs:
  test_linux_chromium_headless_new:
    name: Linux Chromium Headless New
    runs-on: ubuntu-latest
+    env:
+      PWTEST_BOT_NAME: "headless-new"
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-node@v3
@ -833,7 +872,6 @@ jobs:
    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium
      env:
        PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW: 1
-        PWTEST_BOT_NAME: "headless-new"
    - run: node tests/config/checkCoverage.js chromium
    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
      if: always()
@ -843,3 +881,4 @@ jobs:
      uses: ./.github/actions/upload-blob-report
      with:
        report_dir: blob-report
+        job_name: ${{ env.PWTEST_BOT_NAME }}