devops: refactor GitHub Actions workflows (#30524)

.github/actions/run-test/action.yml

@@ -0,0 +1,87 @@
+name: 'Run browser tests'
+description: 'Run browser tests'
+inputs:
+  command:
+    description: 'Command to run tests'
+    required: true
+  node-version:
+    description: 'Node.js version to use'
+    required: false
+    default: '18'
+  browsers-to-install:
+    description: 'Browser to install. Default is all browsers.'
+    required: false
+    default: ''
+  bot-name:
+    description: 'Bot name'
+    required: true
+  shell:
+    description: 'Shell to use'
+    required: false
+    default: 'bash'
+  flakiness-client-id:
+    description: 'Azure Flakiness Dashboard Client ID'
+    required: false
+  flakiness-tenant-id:
+    description: 'Azure Flakiness Dashboard Tenant ID'
+    required: false
+  flakiness-subscription-id:
+    description: 'Azure Flakiness Dashboard Subscription ID'
+    required: false
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-node@v4
+      with:
+        node-version: ${{ inputs.node-version }}
+    # https://github.com/actions/runner-images/issues/9330
+    - name: Allow microphone access to all apps (macOS 14)
+      shell: bash
+      run: |
+        if [[ "$(uname)" == "Darwin" && "$(sw_vers -productVersion | cut -d. -f1)" == "14" ]]; then
+          echo "Allowing microphone access to all apps"
+          sqlite3 $HOME/Library/Application\ Support/com.apple.TCC/TCC.db "INSERT OR IGNORE INTO access VALUES ('kTCCServiceMicrophone','/usr/local/opt/runner/provisioner/provisioner',1,2,4,1,NULL,NULL,0,'UNUSED',NULL,0,1687786159,NULL,NULL,'UNUSED',1687786159);"
+        fi
+    - run: npm ci
+      shell: bash
+      env:
+        DEBUG: pw:install
+        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
+    - run: npm run build
+      shell: bash
+    - run: npx playwright install --with-deps ${{ inputs.browsers-to-install }}
+      shell: bash
+    - name: Run tests
+      if: inputs.shell == 'bash'
+      run: |
+        if [[ "$(uname)" == "Linux" ]]; then
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- ${{ inputs.command }}
+        else
+          ${{ inputs.command }}
+        fi
+      shell: bash
+      env:
+        PWTEST_BOT_NAME: ${{ inputs.bot-name }}
+    - name: Run tests
+      if: inputs.shell != 'bash'
+      run: ${{ inputs.command }}
+      shell: ${{ inputs.shell }}
+      env:
+        PWTEST_BOT_NAME: ${{ inputs.bot-name }}
+    - name: Azure Login
+      uses: azure/login@v2
+      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
+      with:
+        client-id: ${{ inputs.flakiness-client-id }}
+        tenant-id: ${{ inputs.flakiness-tenant-id }}
+        subscription-id: ${{ inputs.flakiness-subscription-id }}
+    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
+      if: ${{ !cancelled() }}
+      shell: bash
+    - name: Upload blob report
+      if: ${{ !cancelled() }}
+      uses: ./.github/actions/upload-blob-report
+      with:
+        report_dir: blob-report
+        job_name: ${{ inputs.bot-name }}

.github/workflows/tests_primary.yml

@@ -42,39 +42,20 @@ jobs:
             node-version: 20
             browser: chromium
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
+    - uses: ./.github/actions/run-test
       with:
         node-version: ${{ matrix.node-version }}
-    - run: npm ci
+        browsers-to-install: ${{ matrix.browser }} chromium
-      env:
+        command: npm run test -- --project=${{ matrix.browser }}-*
-        DEBUG: pw:install
+        bot-name: "${{ matrix.browser }}-${{ matrix.os }}-node${{ matrix.node-version }}"
-        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+        flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-    - run: npm run build
+        flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-    - run: npx playwright install --with-deps ${{ matrix.browser }} chromium
+        flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}-*
-    - name: Azure Login
-      uses: azure/login@v2
-      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
-      with:
-        client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
-      if: ${{ !cancelled() }}
-      shell: bash
-    - name: Upload blob report
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/upload-blob-report
-      with:
-        report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
 
   test_linux_chromium_tot:
     name: ${{ matrix.os }} (chromium tip-of-tree)
@@ -84,42 +65,21 @@ jobs:
       matrix:
         os: [ubuntu-20.04]
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.os }}-chromium-tip-of-tree"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
+    - uses: ./.github/actions/run-test
       with:
-        node-version: 18
+        browsers-to-install: chromium-tip-of-tree
-    - run: npm ci
+        command: npm run test -- --project=chromium-*
-      env:
+        bot-name: "${{ matrix.os }}-chromium-tip-of-tree"
-        DEBUG: pw:install
+        flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+        flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-    - run: npm run build
+        flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: npx playwright install --with-deps chromium-tip-of-tree
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium-*
       env:
         PWTEST_CHANNEL: chromium-tip-of-tree
-        PWTEST_BOT_NAME: "${{ matrix.os }}-chromium-tip-of-tree"
-    - name: Azure Login
-      uses: azure/login@v2
-      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
-      with:
-        client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
-      if: ${{ !cancelled() }}
-      shell: bash
-    - name: Upload blob report
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/upload-blob-report
-      with:
-        report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
 
   test_test_runner:
     name: Test Runner
@@ -149,41 +109,21 @@ jobs:
             shardIndex: 2
             shardTotal: 2
     runs-on: ${{ matrix.os }}
-    env:
-      PWTEST_BOT_NAME: "${{ matrix.os }}-node${{ matrix.node-version }}-${{ matrix.shardIndex }}"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
+    - uses: ./.github/actions/run-test
       with:
         node-version: ${{matrix.node-version}}
-    - run: npm ci
+        command: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        bot-name: "${{ matrix.os }}-node${{ matrix.node-version }}-${{ matrix.shardIndex }}"
+        flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
+        flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
+        flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
       env:
-        DEBUG: pw:install
+        PWTEST_CHANNEL: firefox-beta
-    - run: npm run build
-    - run: npx playwright install --with-deps
-    - run: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
-      if: matrix.os != 'ubuntu-latest'
-    - run: xvfb-run npm run ttest --  --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
-      if: matrix.os == 'ubuntu-latest'
-    - name: Azure Login
-      uses: azure/login@v2
-      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
-      with:
-        client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
-      if: ${{ !cancelled() }}
-      shell: bash
-    - name: Upload blob report
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/upload-blob-report
-      with:
-        report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}
 
   test_web_components:
     name: Web Components
@@ -268,40 +208,19 @@ jobs:
         - windows-latest
     runs-on: ${{ matrix.os  }}
     timeout-minutes: 30
-    env:
-      PWTEST_BOT_NAME: "package-installations-${{ matrix.os }}"
     permissions:
       id-token: write   # This is required for OIDC login (azure/login) to succeed
       contents: read    # This is required for actions/checkout to succeed
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-node@v4
-      with:
-        node-version: 18
-    - run: npm ci
-      env:
-        DEBUG: pw:install
-    - run: npm run build
-    - run: npx playwright install --with-deps
     - run: npm install -g yarn@1
     - run: npm install -g pnpm@8
-    - run: npm run itest
+    - uses: ./.github/actions/run-test
-      if: matrix.os != 'ubuntu-latest'
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run itest
-      if: matrix.os == 'ubuntu-latest'
-    - name: Azure Login
-      uses: azure/login@v2
-      if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }}
       with:
-        client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
+        command: npm run itest
-        tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
+        bot-name: "package-installations-${{ matrix.os }}"
-        subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
+        # TODO: figure out why itest fails with 'bash' on Windows.
-    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
+        shell: ${{ matrix.os == 'windows-latest' && 'pwsh' || 'bash' }}
-      if: ${{ !cancelled() }}
+        flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
-      shell: bash
+        flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }}
-    - name: Upload blob report
+        flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/upload-blob-report
-      with:
-        report_dir: blob-report
-        job_name: ${{ env.PWTEST_BOT_NAME }}