feat(devops): publish merged reports to pull requests (#23419)

* Moved report merging and publishing logic into create_test_report.yml
shared between all workflows
* Merged reports are now published for try jobs on pull requests too. In
order to achieve that the logic had to be extracted into a separate
workflow triggered by
[workflow_run](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_run),
this way it can access secrets even if the original workflow was not
able to.
* The blob report data flow is different depending on whether the
workflow is triggered by a pull request or a push:
- For `pull_request` the workflow doesn't have access to the secrets it
uploads the blob report to the GitHub
artifact storage. Later on the merge workflow uploads that blob report
to Azure blob storage.
- Workflows triggered by `push` event can read secrets. They upload blob
report directly to Azure blob storage
and the merge workflow downloads the report from there rather than from
GitHub artifacts.

.github/actions/download-artifact/action.yml

@@ -0,0 +1,40 @@
+name: 'Download blob report'
+description: 'Download blob report from GitHub artifacts'
+inputs:
+  name:
+    description: 'Name of the artifact to download'
+    required: true
+    type: string
+    default: 'blob-report'
+  path:
+    description: 'Directory with downloaded artifacts'
+    required: true
+    type: string
+    default: 'blob-report'
+runs:
+  using: "composite"
+  steps:
+    - name: Download blob report
+      uses: actions/github-script@v6
+      with:
+        script: |
+          console.log(`downloading artifacts for workflow_run: ${context.payload.workflow_run.id}`);
+          console.log(`workflow_run: ${JSON.stringify(context.payload.workflow_run, null, 2)}`);
+          const { data } = await github.rest.actions.listWorkflowRunArtifacts({
+            ...context.repo,
+            run_id: context.payload.workflow_run.id
+          });
+          console.log('total = ', data.total_count);
+          const name = '${{ inputs.name }}';
+          const report = data.artifacts.filter(a => a.name === name)[0];
+          const result = await github.rest.actions.downloadArtifact({
+            ...context.repo,
+            artifact_id: report.id,
+            archive_format: 'zip'
+          });
+          console.log('download result', result);
+          const fs = require('fs');
+          fs.writeFileSync(`${name}.zip`, Buffer.from(result.data));
+    - name: Unzip blob report
+      shell: bash
+      run: unzip ${{ inputs.name }}.zip -d ${{ inputs.path }}

.github/actions/download-blob-report-from-azure/action.yml

@@ -1,10 +1,15 @@
 name: 'Download blob report from Azure'
 description: 'Download blob report from Azure blob storage'
 inputs:
-  run_dir:
+  blob_prefix:
     description: 'Name of the Azure blob storage directory containing blob report'
     required: true
     type: string
+  output_dir:
+    description: 'Output directory where downloaded blobs will be stored'
+    required: true
+    type: string
+    default: 'blob-report'
   connection_string:
     description: 'Azure connection string'
     required: true
@@ -12,12 +17,13 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Download blob report from Azure
+    - name: Download Blob Reports from Azure Blob Storage
       shell: bash
       run: |
-        mkdir -p ${{ inputs.run_dir }}
-        LIST=$(az storage blob list -c '$web' --prefix ${{ inputs.run_dir }} --connection-string  "${{ inputs.connection_string }}")
+        OUTPUT_DIR='${{ inputs.output_dir }}'
+        mkdir -p $OUTPUT_DIR
+        LIST=$(az storage blob list -c '$web' --prefix ${{ inputs.blob_prefix }} --connection-string  "${{ inputs.connection_string }}")
         for name in $(echo $LIST | jq --raw-output '.[].name | select(test(".jsonl$"))');
         do
-          az storage blob download -c '$web' --name $name -f $name  --connection-string  "${{ inputs.connection_string }}"
+          az storage blob download -c '$web' --name $name -f $OUTPUT_DIR/$(basename $name) --connection-string  "${{ inputs.connection_string }}"
         done

.github/workflows/create_test_report.yml

@@ -0,0 +1,100 @@
+name: Publish Test Results
+on:
+  workflow_run:
+    workflows: ["tests 1", "tests 2"]
+    types:
+      - completed
+jobs:
+  merge-reports:
+    permissions:
+      pull-requests: write
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+
+    - run: npm ci
+      env:
+        DEBUG: pw:install
+        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+
+    - name: Download blob report artifact
+      if: ${{ always() && github.event.workflow_run.event == 'pull_request' }}
+      uses: ./.github/actions/download-artifact
+      with:
+        name: 'blob-report-${{ github.event.workflow_run.run_attempt }}'
+        path: 'blob-report'
+
+    - name: Download blob report from Azure
+      if: ${{ always() && github.event.workflow_run.event == 'push' }}
+      uses: ./.github/actions/download-blob-report-from-azure
+      with:
+        blob_prefix: 'run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}'
+        output_dir: 'blob-report'
+        connection_string: '${{ secrets.AZURE_CONNECTION_STRING }}'
+
+    - name: Merge into HTML Report
+      run: |
+        npx playwright merge-reports --reporter html --attachments missing blob-report
+
+    - name: Upload HTML report to Azure
+      run: |
+        REPORT_DIR='run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}'
+        az storage blob upload-batch -s playwright-report -d "\$web/$REPORT_DIR" --connection-string "${{ secrets.AZURE_CONNECTION_STRING }}"
+        echo "Report url: https://pwblobreport01.z1.web.core.windows.net/$REPORT_DIR/index.html"
+
+    - name: Upload blob report to Azure
+      if: ${{ github.event.workflow_run.event == 'pull_request' }}
+      run: |
+        REPORT_DIR='run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}'
+        az storage blob upload-batch -s blob-report -d "\$web/$REPORT_DIR" --connection-string "${{ secrets.AZURE_CONNECTION_STRING }}"
+
+    - name: Comment on PR
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          let prs = [];
+          if (context.payload.workflow_run.event === 'pull_request') {
+            prs = context.payload.workflow_run.pull_requests;
+          } else if (context.payload.workflow_run.event === 'push') {
+            const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+              ...context.repo,
+              commit_sha: context.sha,
+            });
+            prs = data;
+          } else {
+            core.error('Unsupported workflow trigger event: ' + context.payload.workflow_run.event);
+            return;
+          }
+          if (prs.length === 0) {
+            core.error('No pull request found for commit ' + context.sha + ' and workflow triggered by: ' + context.payload.workflow_run.event);
+            return;
+          }
+          const reportDir = 'run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}';
+          const reportUrl = `https://pwblobreport01.z1.web.core.windows.net/${reportDir}/index.html`;
+          core.notice('Report url: ' + reportUrl);
+          const { data: response } = await github.rest.issues.createComment({
+            ...context.repo,
+            issue_number: prs[0].number,
+            body: `"${{ github.event.workflow_run.name }}" [report](${reportUrl}).`,
+          });
+          core.info('Posted comment: ' + response.html_url);
+
+          if (context.payload.workflow_run.event === 'push') {
+            const check = await github.rest.checks.create({
+              ...context.repo,
+              name: '${{ github.event.workflow_run.name }}',
+              head_sha: context.sha,
+              status: 'completed',
+              conclusion: 'success',
+              details_url: reportUrl,
+              output: {
+                title: '"${{ github.event.workflow_run.name }}" test results',
+                summary: '"${{ github.event.workflow_run.name }}" test results: ' + reportUrl,
+              }
+            });
+          }

.github/workflows/tests_primary.yml

@@ -61,77 +61,16 @@ jobs:
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
-    - name: Upload artifacts to Azure Blob Storage
-      if: always() && github.event_name != 'pull_request'
-      run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
-
-  merge_test_linux:
-    if: ${{ always() && github.event_name != 'pull_request' }}
-    permissions:
-      pull-requests: write
-      checks: write
-    needs: [test_linux]
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-node@v3
+    - name: Upload blob report to Azure
+      if: always() && github.event_name == 'push'
+      run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.sha }}' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+    - name: Upload blob report to GitHub
+      uses: actions/upload-artifact@v3
+      if: always() && github.event_name == 'pull_request'
       with:
-        node-version: 16
-    - run: npm ci
-      env:
-        DEBUG: pw:install
-        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-    - run: npm run build
-    - name: Download blob report from Azure
-      uses: ./.github/actions/download-blob-report-from-azure
-      with:
-        run_dir: 'run-${{ github.run_id }}-${{ github.sha }}-test_linux'
-        connection_string: '${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}'
-    - name: Merge into HTML Report
-      run: |
-        npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_linux'
-    - name: Upload HTML Report to Azure
-      run: |
-        az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
-        echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-test_linux/index.html"
-    - name: Comment on PR
-      uses: actions/github-script@v6
-      with:
-        github-token: ${{secrets.GITHUB_TOKEN}}
-        script: |
-          const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
-            ...context.repo,
-            commit_sha: context.sha,
-          });
-          if (prs.length === 0) {
-            core.error('No pull request found for commit ' + context.sha);
-            return;
-          }
-          const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/run-${context.runId}-${context.sha}-test_linux/index.html`;
-          core.notice('Report url: ' + reportUrl);
-          const { data: response } = await github.rest.issues.createComment({
-            ...context.repo,
-            issue_number: prs[0].number,
-            body: 'Primary test results: ' + reportUrl,
-          });
-          core.info('Posted comment: ' + response.html_url);
-
-          const { owner, repo } = context.repo;
-
-          const check = await github.rest.checks.create({
-            owner,
-            repo,
-            name: 'Primary Tests',
-            head_sha: context.sha,
-            status: 'completed',
-            conclusion: 'success',
-            details_url: reportUrl,
-            output: {
-              title: 'Merged Primary test results',
-              summary: 'Primary test results: ' + reportUrl,
-            }
-          });
+        name: blob-report-${{ github.run_attempt }}
+        path: blob-report
+        retention-days: 30
 
   test_linux_chromium_tot:
     name: ${{ matrix.os }} (chromium tip-of-tree)
@@ -203,77 +142,16 @@ jobs:
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
-    - name: Upload artifacts to Azure Blob Storage
-      if: always() && github.event_name != 'pull_request'
-      run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
-
-  merge_test_test_runner:
-    if: ${{ always() && github.event_name != 'pull_request' }}
-    permissions:
-      pull-requests: write
-      checks: write
-    needs: [test_test_runner]
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-node@v3
+    - name: Upload blob report to Azure
+      if: always() && github.event_name == 'push'
+      run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.sha }}' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+    - name: Upload blob report to GitHub
+      uses: actions/upload-artifact@v3
+      if: always() && github.event_name == 'pull_request'
       with:
-        node-version: 16
-    - run: npm ci
-      env:
-        DEBUG: pw:install
-        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-    - run: npm run build
-    - name: Download blob report from Azure
-      uses: ./.github/actions/download-blob-report-from-azure
-      with:
-        run_dir: 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner'
-        connection_string: '${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}'
-    - name: Merge into HTML Report
-      run: |
-        npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner'
-    - name: Upload HTML report to Azure
-      run: |
-        az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
-        echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner/index.html"
-    - name: Comment on PR
-      uses: actions/github-script@v6
-      with:
-        github-token: ${{secrets.GITHUB_TOKEN}}
-        script: |
-          const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
-            ...context.repo,
-            commit_sha: context.sha,
-          });
-          if (prs.length === 0) {
-            core.error('No pull request found for commit ' + context.sha);
-            return;
-          }
-          const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/run-${context.runId}-${context.sha}-test_test_runner/index.html`;
-          core.notice('Report url: ' + reportUrl);
-          const { data: response } = await github.rest.issues.createComment({
-            ...context.repo,
-            issue_number: prs[0].number,
-            body: 'Test Runner test results: ' + reportUrl,
-          });
-          core.info('Posted comment: ' + response.html_url);
-
-          const { owner, repo } = context.repo;
-
-          const check = await github.rest.checks.create({
-            owner,
-            repo,
-            name: 'Test Runner Tests',
-            head_sha: context.sha,
-            status: 'completed',
-            conclusion: 'success',
-            details_url: reportUrl,
-            output: {
-              title: 'Merged Test Runner Results',
-              summary: 'Test Runner test results: ' + reportUrl,
-            }
-          });
+        name: blob-report-${{ github.run_attempt }}
+        path: blob-report
+        retention-days: 30
 
   test_web_components:
     name: Web Components

.github/workflows/tests_secondary.yml

@@ -208,77 +208,16 @@ jobs:
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
-    - name: Upload artifacts to Azure Blob Storage
-      if: always() && github.event_name != 'pull_request'
-      run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-tracing_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
-
-  merge_tracing_linux:
-    if: ${{ always() && github.event_name != 'pull_request' }}
-    permissions:
-      pull-requests: write
-      checks: write
-    needs: [tracing_linux]
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-node@v3
+    - name: Upload blob report to Azure
+      if: always() && github.event_name == 'push'
+      run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.sha }}' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+    - name: Upload blob report to GitHub
+      uses: actions/upload-artifact@v3
+      if: always() && github.event_name == 'pull_request'
       with:
-        node-version: 16
-    - run: npm ci
-      env:
-        DEBUG: pw:install
-        PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-    - run: npm run build
-    - name: Download blob report from Azure
-      uses: ./.github/actions/download-blob-report-from-azure
-      with:
-        run_dir: 'run-${{ github.run_id }}-${{ github.sha }}-tracing_linux'
-        connection_string: '${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}'
-    - name: Merge into HTML Report
-      run: |
-        npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-tracing_linux'
-    - name: Upload HTML report to Azure
-      run: |
-        az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-tracing_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
-        echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-tracing_linux/index.html"
-    - name: Comment on PR
-      uses: actions/github-script@v6
-      with:
-        github-token: ${{secrets.GITHUB_TOKEN}}
-        script: |
-          const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
-            ...context.repo,
-            commit_sha: context.sha,
-          });
-          if (prs.length === 0) {
-            core.error('No pull request found for commit ' + context.sha);
-            return;
-          }
-          const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/run-${context.runId}-${context.sha}-tracing_linux/index.html`;
-          core.notice('Report url: ' + reportUrl);
-          const { data: response } = await github.rest.issues.createComment({
-            ...context.repo,
-            issue_number: prs[0].number,
-            body: 'Tracing test results: ' + reportUrl,
-          });
-          core.info('Posted comment: ' + response.html_url);
-
-          const { owner, repo } = context.repo;
-
-          const check = await github.rest.checks.create({
-            owner,
-            repo,
-            name: 'Tracing Tests',
-            head_sha: context.sha,
-            status: 'completed',
-            conclusion: 'success',
-            details_url: reportUrl,
-            output: {
-              title: 'Merged Tracing test results',
-              summary: 'Tracing test results: ' + reportUrl,
-            }
-          });
+        name: blob-report-${{ github.run_attempt }}
+        path: blob-report
+        retention-days: 30
 
   chrome_stable_linux:
     name: "Chrome Stable (Linux)"