diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cb488910be..c6232c1371 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,6 +53,7 @@ jobs: fail-fast: false matrix: browser: [chromium, firefox, webkit] + shard: ['1', '2'] runs-on: macos-latest steps: - uses: actions/checkout@v2 @@ -62,14 +63,14 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: npm run build - - run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json + - run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json --shard=${{ matrix.shard }}/2 env: BROWSER: ${{ matrix.browser }} PWRUNNER_JSON_REPORT: "test-results/report.json" - uses: actions/upload-artifact@v1 if: ${{ always() }} with: - name: ${{ matrix.browser }}-mac-test-results + name: ${{ matrix.browser }}-mac-test-results-${{ matrix.shard }} path: test-results test_win: diff --git a/package-lock.json b/package-lock.json index 894246264b..df82f02b8e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1187,9 +1187,9 @@ } }, "@playwright/test-runner": { - "version": "0.3.17", - "resolved": "https://registry.npmjs.org/@playwright/test-runner/-/test-runner-0.3.17.tgz", - "integrity": "sha512-XxMkZTBim9PvmWmUUM0h3zIitwp4RqyKVnc662nNDHmMTqAdGNB+z4OmF6f9wT+GjY3VPc8gfxlDaO6vXvx1JA==", + "version": "0.3.18", + "resolved": "https://registry.npmjs.org/@playwright/test-runner/-/test-runner-0.3.18.tgz", + "integrity": "sha512-QvkStde6VE6pV0oukwV/U2P1aT96sVndbaMZAmXx2w31HotOl2w/G4tWVAIV/UtKtcI3fFY2womfUB3wUAAyjw==", "dev": true, "requires": { "@babel/code-frame": "^7.10.4", @@ -2321,9 +2321,9 @@ "dev": true }, "caniuse-lite": { - "version": "1.0.30001131", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001131.tgz", - "integrity": "sha512-4QYi6Mal4MMfQMSqGIRPGbKIbZygeN83QsWq1ixpUwvtfgAZot5BrCKzGygvZaV+CnELdTwD0S4cqUNozq7/Cw==", + "version": "1.0.30001133", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001133.tgz", + "integrity": "sha512-s3XAUFaC/ntDb1O3lcw9K8MPeOW7KO3z9+GzAoBxfz1B0VdacXPMKgFUtG4KIsgmnbexmi013s9miVu4h+qMHw==", "dev": true }, "caseless": { @@ -4556,9 +4556,9 @@ "dev": true }, "is-callable": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.1.tgz", - "integrity": "sha512-wliAfSzx6V+6WfMOmus1xy0XvSgf/dlStkvTfq7F0g4bOIW0PSUbnyse3NhDwdyYS1ozfUtAAySqTws3z9Eqgg==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz", + "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==", "dev": true }, "is-data-descriptor": { diff --git a/package.json b/package.json index 34018dbca1..27be14e47b 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "ws": "^7.3.1" }, "devDependencies": { - "@playwright/test-runner": "^0.3.17", + "@playwright/test-runner": "^0.3.18", "@types/debug": "^4.1.5", "@types/extract-zip": "^1.6.2", "@types/mime": "^2.0.3", diff --git a/test/browsertype-launch-server.spec.ts b/test/browsertype-launch-server.spec.ts index 26c4b8ceb8..38cde55ff7 100644 --- a/test/browsertype-launch-server.spec.ts +++ b/test/browsertype-launch-server.spec.ts @@ -26,9 +26,9 @@ describe('lauch server', suite => { await browserServer.close(); }); - it('should work with port', async ({browserType, defaultBrowserOptions, parallelIndex}) => { - const browserServer = await browserType.launchServer({ ...defaultBrowserOptions, port: 8800 + parallelIndex }); - expect(browserServer.wsEndpoint()).toContain(String(8800 + parallelIndex)); + it('should work with port', async ({browserType, defaultBrowserOptions, workerIndex}) => { + const browserServer = await browserType.launchServer({ ...defaultBrowserOptions, port: 8800 + workerIndex }); + expect(browserServer.wsEndpoint()).toContain(String(8800 + workerIndex)); await browserServer.close(); }); diff --git a/test/playwright.fixtures.ts b/test/playwright.fixtures.ts index 2be50d8081..b23c109af0 100644 --- a/test/playwright.fixtures.ts +++ b/test/playwright.fixtures.ts @@ -97,11 +97,11 @@ export const options = { TRACING: valueFromEnv('TRACING', false), }; -defineWorkerFixture('httpService', async ({parallelIndex}, test) => { +defineWorkerFixture('httpService', async ({workerIndex}, test) => { const assetsPath = path.join(__dirname, 'assets'); const cachedPath = path.join(__dirname, 'assets', 'cached'); - const port = 8907 + parallelIndex * 2; + const port = 8907 + workerIndex * 2; const server = await TestServer.create(assetsPath, port); server.enableHTTPCache(cachedPath); @@ -139,7 +139,7 @@ defineWorkerFixture('defaultBrowserOptions', async ({browserName}, runTest, conf }); }); -defineWorkerFixture('playwright', async ({browserName, parallelIndex, platform}, test) => { +defineWorkerFixture('playwright', async ({browserName, workerIndex, platform}, test) => { assert(platform); // Depend on platform to generate all tests. const {coverage, uninstall} = installCoverageHooks(browserName); if (options.WIRE) { @@ -172,7 +172,7 @@ defineWorkerFixture('playwright', async ({browserName, parallelIndex, platform}, async function teardownCoverage() { uninstall(); - const coveragePath = path.join(__dirname, 'coverage-report', parallelIndex + '.json'); + const coveragePath = path.join(__dirname, 'coverage-report', workerIndex + '.json'); const coverageJSON = [...coverage.keys()].filter(key => coverage.get(key)); await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true }); await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8'); @@ -260,10 +260,9 @@ defineWorkerFixture('expectedSSLError', async ({browserName, platform}, runTest) }); defineTestFixture('testOutputDir', async ({}, runTest, info) => { - const { spec, config } = info; - const relativePath = path.relative(config.testDir, spec.file).replace(/\.spec\.[jt]s/, ''); - const sanitizedTitle = spec.title.replace(/[^\w\d]+/g, '_'); - const testOutputDir = path.join(config.outputDir, relativePath, sanitizedTitle); + const relativePath = path.relative(info.config.testDir, info.file).replace(/\.spec\.[jt]s/, ''); + const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_'); + const testOutputDir = path.join(info.config.outputDir, relativePath, sanitizedTitle); await fs.promises.mkdir(testOutputDir, { recursive: true }); await runTest(testOutputDir); const files = await fs.promises.readdir(testOutputDir); @@ -291,8 +290,7 @@ defineTestFixture('context', async ({browser, testOutputDir}, runTest, info) => defineTestFixture('page', async ({context, testOutputDir}, runTest, info) => { const page = await context.newPage(); await runTest(page); - const { testRun } = info; - if (testRun.status === 'failed' || testRun.status === 'timedOut') + if (info.status === 'failed' || info.status === 'timedOut') await page.screenshot({ timeout: 5000, path: path.join(testOutputDir, 'test-failed.png') }); }); diff --git a/test/proxy.spec.ts b/test/proxy.spec.ts index be413b5a4b..f7d2d23b52 100644 --- a/test/proxy.spec.ts +++ b/test/proxy.spec.ts @@ -121,7 +121,7 @@ it('should exclude patterns', (test, parameters) => { it('should use socks proxy', (test, parameters) => { test.flaky(options.MAC(parameters) && options.WEBKIT(parameters), 'Intermittent page.goto: The network connection was lost error on bots'); -}, async ({ browserType, defaultBrowserOptions, parallelIndex }) => { +}, async ({ browserType, defaultBrowserOptions, workerIndex }) => { const server = socks.createServer((info, accept, deny) => { let socket; if ((socket = accept(true))) { @@ -138,7 +138,7 @@ it('should use socks proxy', (test, parameters) => { ].join('\r\n')); } }); - const socksPort = 9107 + parallelIndex * 2; + const socksPort = 9107 + workerIndex * 2; server.listen(socksPort, 'localhost'); server.useAuth(socks.auth.None()); diff --git a/test/screencast.spec.ts b/test/screencast.spec.ts index ff41843526..81da6cf532 100644 --- a/test/screencast.spec.ts +++ b/test/screencast.spec.ts @@ -58,8 +58,7 @@ defineTestFixture('videoPlayer', async ({videoPlayerBrowser, server}, test) => { }); defineTestFixture('relativeArtifactsPath', async ({browserType}, runTest, info) => { - const { spec } = info; - const sanitizedTitle = spec.title.replace(/[^\w\d]+/g, '_'); + const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_'); const relativeArtifactsPath = `${browserType.name()}-${sanitizedTitle}`; await runTest(relativeArtifactsPath); });