diff --git a/.github/actions/run-test/action.yml b/.github/actions/run-test/action.yml index 568e3900bb..dcf8168367 100644 --- a/.github/actions/run-test/action.yml +++ b/.github/actions/run-test/action.yml @@ -85,7 +85,8 @@ runs: if: ${{ !cancelled() }} shell: bash - name: Upload blob report - if: ${{ !cancelled() }} + # We only merge reports for PRs as per .github/workflows/create_test_report.yml. + if: ${{ !cancelled() && github.event.workflow_run.event == 'pull_request' }} uses: ./.github/actions/upload-blob-report with: report_dir: blob-report diff --git a/.github/workflows/create_test_report.yml b/.github/workflows/create_test_report.yml index 8ae54609e4..9d382f1ee6 100644 --- a/.github/workflows/create_test_report.yml +++ b/.github/workflows/create_test_report.yml @@ -1,7 +1,7 @@ name: Publish Test Results on: workflow_run: - workflows: ["tests 1", "tests 2"] + workflows: ["tests 1", "tests 2", "tests others"] types: - completed jobs: diff --git a/.github/workflows/tests_clock.yml b/.github/workflows/tests_clock.yml deleted file mode 100644 index cc9a519ee3..0000000000 --- a/.github/workflows/tests_clock.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: "tests Clock" - -on: - push: - branches: - - main - - release-* - pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - types: [ labeled ] - branches: - - main - - release-* - -env: - # Force terminal colors. @see https://www.npmjs.com/package/colors - FORCE_COLOR: 1 - ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - -jobs: - frozen_time_linux: - name: Frozen time library - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - node-version: 20 - browsers-to-install: chromium - command: npm run test -- --project=chromium-* - bot-name: "frozen-time-library-chromium-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PW_FREEZE_TIME: 1 - - frozen_time_test_runner: - name: Frozen time test runner - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ubuntu-22.04 - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - node-version: 20 - command: npm run ttest - bot-name: "frozen-time-runner-chromium-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PW_FREEZE_TIME: 1 diff --git a/.github/workflows/tests_electron.yml b/.github/workflows/tests_electron.yml deleted file mode 100644 index 604c2e0e2c..0000000000 --- a/.github/workflows/tests_electron.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: "electron" - -on: - push: - branches: - - main - - release-* - pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - types: [ labeled ] - branches: - - main - - release-* - -env: - # Force terminal colors. @see https://www.npmjs.com/package/colors - FORCE_COLOR: 1 - -jobs: - test_electron: - name: ${{ matrix.os }} - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: chromium - command: npm run etest - bot-name: "electron-${{ matrix.os }}" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} diff --git a/.github/workflows/tests_others.yml b/.github/workflows/tests_others.yml new file mode 100644 index 0000000000..04115ce693 --- /dev/null +++ b/.github/workflows/tests_others.yml @@ -0,0 +1,153 @@ +name: tests others + +on: + push: + branches: + - main + - release-* + pull_request: + paths-ignore: + - 'browser_patches/**' + - 'docs/**' + types: [ labeled ] + branches: + - main + - release-* + +env: + FORCE_COLOR: 1 + ELECTRON_SKIP_BINARY_DOWNLOAD: 1 + +jobs: + test_stress: + name: Stress - ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Stick with macos-latest-large for now which is Intel-based until + # https://github.com/microsoft/playwright/issues/30705 is fixed. + os: [ubuntu-latest, macos-latest-large, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - run: npm ci + - run: npm run build + - run: npx playwright install --with-deps + - run: npm run stest contexts -- --project=chromium + if: ${{ !cancelled() }} + - run: npm run stest browsers -- --project=chromium + if: ${{ !cancelled() }} + - run: npm run stest frames -- --project=chromium + if: ${{ !cancelled() }} + - run: npm run stest contexts -- --project=webkit + if: ${{ !cancelled() }} + - run: npm run stest browsers -- --project=webkit + if: ${{ !cancelled() }} + - run: npm run stest frames -- --project=webkit + if: ${{ !cancelled() }} + - run: npm run stest contexts -- --project=firefox + if: ${{ !cancelled() }} + - run: npm run stest browsers -- --project=firefox + if: ${{ !cancelled() }} + - run: npm run stest frames -- --project=firefox + if: ${{ !cancelled() }} + - run: npm run stest heap -- --project=chromium + if: ${{ !cancelled() }} + + test_webview2: + name: WebView2 + environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} + runs-on: windows-2022 + permissions: + id-token: write # This is required for OIDC login (azure/login) to succeed + contents: read # This is required for actions/checkout to succeed + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: '8.0.x' + - run: dotnet build + working-directory: tests/webview2/webview2-app/ + - name: Update to Evergreen WebView2 Runtime + shell: pwsh + run: | + # See here: https://developer.microsoft.com/en-us/microsoft-edge/webview2/ + Invoke-WebRequest -Uri 'https://go.microsoft.com/fwlink/p/?LinkId=2124703' -OutFile 'setup.exe' + Start-Process -FilePath setup.exe -Verb RunAs -Wait + - uses: ./.github/actions/run-test + with: + node-version: 20 + browsers-to-install: chromium + command: npm run webview2test + bot-name: "webview2-chromium-windows" + flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} + flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} + flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} + + test_clock_frozen_time_linux: + name: Frozen time library + environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} + permissions: + id-token: write # This is required for OIDC login (azure/login) to succeed + contents: read # This is required for actions/checkout to succeed + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/run-test + with: + node-version: 20 + browsers-to-install: chromium + command: npm run test -- --project=chromium-* + bot-name: "frozen-time-library-chromium-linux" + flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} + flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} + flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} + env: + PW_FREEZE_TIME: 1 + + test_clock_frozen_time_test_runner: + name: Frozen time test runner + environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} + runs-on: ubuntu-22.04 + permissions: + id-token: write # This is required for OIDC login (azure/login) to succeed + contents: read # This is required for actions/checkout to succeed + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/run-test + with: + node-version: 20 + command: npm run ttest + bot-name: "frozen-time-runner-chromium-linux" + flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} + flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} + flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} + env: + PW_FREEZE_TIME: 1 + + test_electron: + name: Electron - ${{ matrix.os }} + environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + permissions: + id-token: write # This is required for OIDC login (azure/login) to succeed + contents: read # This is required for actions/checkout to succeed + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/run-test + with: + browsers-to-install: chromium + command: npm run etest + bot-name: "electron-${{ matrix.os }}" + flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} + flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} + flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} + env: + ELECTRON_SKIP_BINARY_DOWNLOAD: diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index adb1f2b8ed..fd5458fb89 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -307,7 +307,9 @@ jobs: firefox_beta_mac: name: "Firefox Beta (Mac)" environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: macos-latest + # Stick with macos-latest-large for now which is Intel-based until + # https://github.com/microsoft/playwright/issues/30705 is fixed. + runs-on: macos-latest-large steps: - uses: actions/checkout@v4 - uses: ./.github/actions/run-test diff --git a/.github/workflows/tests_stress.yml b/.github/workflows/tests_stress.yml deleted file mode 100644 index 6001f04567..0000000000 --- a/.github/workflows/tests_stress.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: "stress" - -on: - push: - branches: - - main - - release-* - pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - types: [ labeled ] - branches: - - main - - release-* - -env: - FORCE_COLOR: 1 - ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - -jobs: - test_components: - name: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - - run: npm ci - - run: npm run build - - run: npx playwright install --with-deps - - run: npm run stest contexts -- --project=chromium - if: ${{ !cancelled() }} - - run: npm run stest browsers -- --project=chromium - if: ${{ !cancelled() }} - - run: npm run stest frames -- --project=chromium - if: ${{ !cancelled() }} - - run: npm run stest contexts -- --project=webkit - if: ${{ !cancelled() }} - - run: npm run stest browsers -- --project=webkit - if: ${{ !cancelled() }} - - run: npm run stest frames -- --project=webkit - if: ${{ !cancelled() }} - - run: npm run stest contexts -- --project=firefox - if: ${{ !cancelled() }} - - run: npm run stest browsers -- --project=firefox - if: ${{ !cancelled() }} - - run: npm run stest frames -- --project=firefox - if: ${{ !cancelled() }} - - run: npm run stest heap -- --project=chromium - if: ${{ !cancelled() }} diff --git a/.github/workflows/tests_webview2.yml b/.github/workflows/tests_webview2.yml deleted file mode 100644 index e894c2dc78..0000000000 --- a/.github/workflows/tests_webview2.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: "WebView2 Tests" - -on: - push: - branches: - - main - - release-* - pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - types: [ labeled ] - branches: - - main - - release-* - -env: - # Force terminal colors. @see https://www.npmjs.com/package/colors - FORCE_COLOR: 1 - ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - -jobs: - test_webview2: - name: WebView2 - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: windows-2022 - permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed - contents: read # This is required for actions/checkout to succeed - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 18 - - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '8.0.x' - - run: npm ci - env: - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - - run: npm run build - - run: dotnet build - working-directory: tests/webview2/webview2-app/ - - name: Update to Evergreen WebView2 Runtime - shell: pwsh - run: | - # See here: https://developer.microsoft.com/en-us/microsoft-edge/webview2/ - Invoke-WebRequest -Uri 'https://go.microsoft.com/fwlink/p/?LinkId=2124703' -OutFile 'setup.exe' - Start-Process -FilePath setup.exe -Verb RunAs -Wait - - run: npm run webview2test - - name: Azure Login - uses: azure/login@v2 - if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }} - with: - client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json - if: ${{ !cancelled() }} - shell: bash diff --git a/tests/electron/playwright.config.ts b/tests/electron/playwright.config.ts index 562e2ae6a1..b630b38e56 100644 --- a/tests/electron/playwright.config.ts +++ b/tests/electron/playwright.config.ts @@ -35,6 +35,7 @@ const config: Config = { reporter: process.env.CI ? [ ['dot'], ['json', { outputFile: path.join(outputDir, 'report.json') }], + ['blob', { fileName: `${process.env.PWTEST_BOT_NAME}.zip` }], ] : 'line', projects: [], globalSetup: './globalSetup.ts' diff --git a/tests/webview2/playwright.config.ts b/tests/webview2/playwright.config.ts index 1bdd704a35..0710378a50 100644 --- a/tests/webview2/playwright.config.ts +++ b/tests/webview2/playwright.config.ts @@ -35,6 +35,7 @@ const config: Config = { reporter: process.env.CI ? [ ['dot'], ['json', { outputFile: path.join(outputDir, 'report.json') }], + ['blob', { fileName: `${process.env.PWTEST_BOT_NAME}.zip` }], ] : 'line', projects: [], globalSetup: './globalSetup.ts',