diff --git a/.github/actions/download-artifact/action.yml b/.github/actions/download-artifact/action.yml new file mode 100644 index 0000000000..874b202c06 --- /dev/null +++ b/.github/actions/download-artifact/action.yml @@ -0,0 +1,40 @@ +name: 'Download blob report' +description: 'Download blob report from GitHub artifacts' +inputs: + name: + description: 'Name of the artifact to download' + required: true + type: string + default: 'blob-report' + path: + description: 'Directory with downloaded artifacts' + required: true + type: string + default: 'blob-report' +runs: + using: "composite" + steps: + - name: Download blob report + uses: actions/github-script@v6 + with: + script: | + console.log(`downloading artifacts for workflow_run: ${context.payload.workflow_run.id}`); + console.log(`workflow_run: ${JSON.stringify(context.payload.workflow_run, null, 2)}`); + const { data } = await github.rest.actions.listWorkflowRunArtifacts({ + ...context.repo, + run_id: context.payload.workflow_run.id + }); + console.log('total = ', data.total_count); + const name = '${{ inputs.name }}'; + const report = data.artifacts.filter(a => a.name === name)[0]; + const result = await github.rest.actions.downloadArtifact({ + ...context.repo, + artifact_id: report.id, + archive_format: 'zip' + }); + console.log('download result', result); + const fs = require('fs'); + fs.writeFileSync(`${name}.zip`, Buffer.from(result.data)); + - name: Unzip blob report + shell: bash + run: unzip ${{ inputs.name }}.zip -d ${{ inputs.path }} diff --git a/.github/actions/download-blob-report-from-azure/action.yml b/.github/actions/download-blob-report-from-azure/action.yml index 8986053183..c2bd197257 100644 --- a/.github/actions/download-blob-report-from-azure/action.yml +++ b/.github/actions/download-blob-report-from-azure/action.yml @@ -1,10 +1,15 @@ name: 'Download blob report from Azure' description: 'Download blob report from Azure blob storage' inputs: - run_dir: + blob_prefix: description: 'Name of the Azure blob storage directory containing blob report' required: true type: string + output_dir: + description: 'Output directory where downloaded blobs will be stored' + required: true + type: string + default: 'blob-report' connection_string: description: 'Azure connection string' required: true @@ -12,12 +17,13 @@ inputs: runs: using: "composite" steps: - - name: Download blob report from Azure + - name: Download Blob Reports from Azure Blob Storage shell: bash run: | - mkdir -p ${{ inputs.run_dir }} - LIST=$(az storage blob list -c '$web' --prefix ${{ inputs.run_dir }} --connection-string "${{ inputs.connection_string }}") + OUTPUT_DIR='${{ inputs.output_dir }}' + mkdir -p $OUTPUT_DIR + LIST=$(az storage blob list -c '$web' --prefix ${{ inputs.blob_prefix }} --connection-string "${{ inputs.connection_string }}") for name in $(echo $LIST | jq --raw-output '.[].name | select(test(".jsonl$"))'); do - az storage blob download -c '$web' --name $name -f $name --connection-string "${{ inputs.connection_string }}" + az storage blob download -c '$web' --name $name -f $OUTPUT_DIR/$(basename $name) --connection-string "${{ inputs.connection_string }}" done \ No newline at end of file diff --git a/.github/workflows/create_test_report.yml b/.github/workflows/create_test_report.yml new file mode 100644 index 0000000000..a42333f522 --- /dev/null +++ b/.github/workflows/create_test_report.yml @@ -0,0 +1,100 @@ +name: Publish Test Results +on: + workflow_run: + workflows: ["tests 1", "tests 2"] + types: + - completed +jobs: + merge-reports: + permissions: + pull-requests: write + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 16 + + - run: npm ci + env: + DEBUG: pw:install + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + + - name: Download blob report artifact + if: ${{ always() && github.event.workflow_run.event == 'pull_request' }} + uses: ./.github/actions/download-artifact + with: + name: 'blob-report-${{ github.event.workflow_run.run_attempt }}' + path: 'blob-report' + + - name: Download blob report from Azure + if: ${{ always() && github.event.workflow_run.event == 'push' }} + uses: ./.github/actions/download-blob-report-from-azure + with: + blob_prefix: 'run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}' + output_dir: 'blob-report' + connection_string: '${{ secrets.AZURE_CONNECTION_STRING }}' + + - name: Merge into HTML Report + run: | + npx playwright merge-reports --reporter html --attachments missing blob-report + + - name: Upload HTML report to Azure + run: | + REPORT_DIR='run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}' + az storage blob upload-batch -s playwright-report -d "\$web/$REPORT_DIR" --connection-string "${{ secrets.AZURE_CONNECTION_STRING }}" + echo "Report url: https://pwblobreport01.z1.web.core.windows.net/$REPORT_DIR/index.html" + + - name: Upload blob report to Azure + if: ${{ github.event.workflow_run.event == 'pull_request' }} + run: | + REPORT_DIR='run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}' + az storage blob upload-batch -s blob-report -d "\$web/$REPORT_DIR" --connection-string "${{ secrets.AZURE_CONNECTION_STRING }}" + + - name: Comment on PR + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + let prs = []; + if (context.payload.workflow_run.event === 'pull_request') { + prs = context.payload.workflow_run.pull_requests; + } else if (context.payload.workflow_run.event === 'push') { + const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + ...context.repo, + commit_sha: context.sha, + }); + prs = data; + } else { + core.error('Unsupported workflow trigger event: ' + context.payload.workflow_run.event); + return; + } + if (prs.length === 0) { + core.error('No pull request found for commit ' + context.sha + ' and workflow triggered by: ' + context.payload.workflow_run.event); + return; + } + const reportDir = 'run-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}-${{ github.sha }}'; + const reportUrl = `https://pwblobreport01.z1.web.core.windows.net/${reportDir}/index.html`; + core.notice('Report url: ' + reportUrl); + const { data: response } = await github.rest.issues.createComment({ + ...context.repo, + issue_number: prs[0].number, + body: `"${{ github.event.workflow_run.name }}" [report](${reportUrl}).`, + }); + core.info('Posted comment: ' + response.html_url); + + if (context.payload.workflow_run.event === 'push') { + const check = await github.rest.checks.create({ + ...context.repo, + name: '${{ github.event.workflow_run.name }}', + head_sha: context.sha, + status: 'completed', + conclusion: 'success', + details_url: reportUrl, + output: { + title: '"${{ github.event.workflow_run.name }}" test results', + summary: '"${{ github.event.workflow_run.name }}" test results: ' + reportUrl, + } + }); + } diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index 939d3de9f3..1d24f0d767 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -61,77 +61,16 @@ jobs: - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash - - name: Upload artifacts to Azure Blob Storage - if: always() && github.event_name != 'pull_request' - run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" - - merge_test_linux: - if: ${{ always() && github.event_name != 'pull_request' }} - permissions: - pull-requests: write - checks: write - needs: [test_linux] - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - name: Upload blob report to Azure + if: always() && github.event_name == 'push' + run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.sha }}' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" + - name: Upload blob report to GitHub + uses: actions/upload-artifact@v3 + if: always() && github.event_name == 'pull_request' with: - node-version: 16 - - run: npm ci - env: - DEBUG: pw:install - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - - run: npm run build - - name: Download blob report from Azure - uses: ./.github/actions/download-blob-report-from-azure - with: - run_dir: 'run-${{ github.run_id }}-${{ github.sha }}-test_linux' - connection_string: '${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}' - - name: Merge into HTML Report - run: | - npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_linux' - - name: Upload HTML Report to Azure - run: | - az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" - echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-test_linux/index.html" - - name: Comment on PR - uses: actions/github-script@v6 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - ...context.repo, - commit_sha: context.sha, - }); - if (prs.length === 0) { - core.error('No pull request found for commit ' + context.sha); - return; - } - const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/run-${context.runId}-${context.sha}-test_linux/index.html`; - core.notice('Report url: ' + reportUrl); - const { data: response } = await github.rest.issues.createComment({ - ...context.repo, - issue_number: prs[0].number, - body: 'Primary test results: ' + reportUrl, - }); - core.info('Posted comment: ' + response.html_url); - - const { owner, repo } = context.repo; - - const check = await github.rest.checks.create({ - owner, - repo, - name: 'Primary Tests', - head_sha: context.sha, - status: 'completed', - conclusion: 'success', - details_url: reportUrl, - output: { - title: 'Merged Primary test results', - summary: 'Primary test results: ' + reportUrl, - } - }); + name: blob-report-${{ github.run_attempt }} + path: blob-report + retention-days: 30 test_linux_chromium_tot: name: ${{ matrix.os }} (chromium tip-of-tree) @@ -203,77 +142,16 @@ jobs: - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash - - name: Upload artifacts to Azure Blob Storage - if: always() && github.event_name != 'pull_request' - run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" - - merge_test_test_runner: - if: ${{ always() && github.event_name != 'pull_request' }} - permissions: - pull-requests: write - checks: write - needs: [test_test_runner] - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - name: Upload blob report to Azure + if: always() && github.event_name == 'push' + run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.sha }}' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" + - name: Upload blob report to GitHub + uses: actions/upload-artifact@v3 + if: always() && github.event_name == 'pull_request' with: - node-version: 16 - - run: npm ci - env: - DEBUG: pw:install - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - - run: npm run build - - name: Download blob report from Azure - uses: ./.github/actions/download-blob-report-from-azure - with: - run_dir: 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' - connection_string: '${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}' - - name: Merge into HTML Report - run: | - npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' - - name: Upload HTML report to Azure - run: | - az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" - echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner/index.html" - - name: Comment on PR - uses: actions/github-script@v6 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - ...context.repo, - commit_sha: context.sha, - }); - if (prs.length === 0) { - core.error('No pull request found for commit ' + context.sha); - return; - } - const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/run-${context.runId}-${context.sha}-test_test_runner/index.html`; - core.notice('Report url: ' + reportUrl); - const { data: response } = await github.rest.issues.createComment({ - ...context.repo, - issue_number: prs[0].number, - body: 'Test Runner test results: ' + reportUrl, - }); - core.info('Posted comment: ' + response.html_url); - - const { owner, repo } = context.repo; - - const check = await github.rest.checks.create({ - owner, - repo, - name: 'Test Runner Tests', - head_sha: context.sha, - status: 'completed', - conclusion: 'success', - details_url: reportUrl, - output: { - title: 'Merged Test Runner Results', - summary: 'Test Runner test results: ' + reportUrl, - } - }); + name: blob-report-${{ github.run_attempt }} + path: blob-report + retention-days: 30 test_web_components: name: Web Components diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index a5a0c35fdc..180469d43f 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -208,77 +208,16 @@ jobs: - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash - - name: Upload artifacts to Azure Blob Storage - if: always() && github.event_name != 'pull_request' - run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-tracing_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" - - merge_tracing_linux: - if: ${{ always() && github.event_name != 'pull_request' }} - permissions: - pull-requests: write - checks: write - needs: [tracing_linux] - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - name: Upload blob report to Azure + if: always() && github.event_name == 'push' + run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.sha }}' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" + - name: Upload blob report to GitHub + uses: actions/upload-artifact@v3 + if: always() && github.event_name == 'pull_request' with: - node-version: 16 - - run: npm ci - env: - DEBUG: pw:install - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - - run: npm run build - - name: Download blob report from Azure - uses: ./.github/actions/download-blob-report-from-azure - with: - run_dir: 'run-${{ github.run_id }}-${{ github.sha }}-tracing_linux' - connection_string: '${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}' - - name: Merge into HTML Report - run: | - npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-tracing_linux' - - name: Upload HTML report to Azure - run: | - az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-tracing_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" - echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-tracing_linux/index.html" - - name: Comment on PR - uses: actions/github-script@v6 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - ...context.repo, - commit_sha: context.sha, - }); - if (prs.length === 0) { - core.error('No pull request found for commit ' + context.sha); - return; - } - const reportUrl = `https://mspwblobreport.z1.web.core.windows.net/run-${context.runId}-${context.sha}-tracing_linux/index.html`; - core.notice('Report url: ' + reportUrl); - const { data: response } = await github.rest.issues.createComment({ - ...context.repo, - issue_number: prs[0].number, - body: 'Tracing test results: ' + reportUrl, - }); - core.info('Posted comment: ' + response.html_url); - - const { owner, repo } = context.repo; - - const check = await github.rest.checks.create({ - owner, - repo, - name: 'Tracing Tests', - head_sha: context.sha, - status: 'completed', - conclusion: 'success', - details_url: reportUrl, - output: { - title: 'Merged Tracing test results', - summary: 'Tracing test results: ' + reportUrl, - } - }); + name: blob-report-${{ github.run_attempt }} + path: blob-report + retention-days: 30 chrome_stable_linux: name: "Chrome Stable (Linux)"