test: roll test runner to 0.3.18 (#3949)

.github/workflows/tests.yml

@@ -53,6 +53,7 @@ jobs:
       fail-fast: false
       matrix:
         browser: [chromium, firefox, webkit]
+        shard: ['1', '2']
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v2
@@ -62,14 +63,14 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json
+    - run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json --shard=${{ matrix.shard }}/2
       env:
         BROWSER: ${{ matrix.browser }}
         PWRUNNER_JSON_REPORT: "test-results/report.json"
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
       with:
-        name: ${{ matrix.browser }}-mac-test-results
+        name: ${{ matrix.browser }}-mac-test-results-${{ matrix.shard }}
         path: test-results
 
   test_win:

package-lock.json

@@ -1187,9 +1187,9 @@
       }
     },
     "@playwright/test-runner": {
-      "version": "0.3.17",
-      "resolved": "https://registry.npmjs.org/@playwright/test-runner/-/test-runner-0.3.17.tgz",
-      "integrity": "sha512-XxMkZTBim9PvmWmUUM0h3zIitwp4RqyKVnc662nNDHmMTqAdGNB+z4OmF6f9wT+GjY3VPc8gfxlDaO6vXvx1JA==",
+      "version": "0.3.18",
+      "resolved": "https://registry.npmjs.org/@playwright/test-runner/-/test-runner-0.3.18.tgz",
+      "integrity": "sha512-QvkStde6VE6pV0oukwV/U2P1aT96sVndbaMZAmXx2w31HotOl2w/G4tWVAIV/UtKtcI3fFY2womfUB3wUAAyjw==",
       "dev": true,
       "requires": {
         "@babel/code-frame": "^7.10.4",
@@ -2321,9 +2321,9 @@
       "dev": true
     },
     "caniuse-lite": {
-      "version": "1.0.30001131",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001131.tgz",
-      "integrity": "sha512-4QYi6Mal4MMfQMSqGIRPGbKIbZygeN83QsWq1ixpUwvtfgAZot5BrCKzGygvZaV+CnELdTwD0S4cqUNozq7/Cw==",
+      "version": "1.0.30001133",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001133.tgz",
+      "integrity": "sha512-s3XAUFaC/ntDb1O3lcw9K8MPeOW7KO3z9+GzAoBxfz1B0VdacXPMKgFUtG4KIsgmnbexmi013s9miVu4h+qMHw==",
       "dev": true
     },
     "caseless": {
@@ -4556,9 +4556,9 @@
       "dev": true
     },
     "is-callable": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.1.tgz",
-      "integrity": "sha512-wliAfSzx6V+6WfMOmus1xy0XvSgf/dlStkvTfq7F0g4bOIW0PSUbnyse3NhDwdyYS1ozfUtAAySqTws3z9Eqgg==",
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz",
+      "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==",
       "dev": true
     },
     "is-data-descriptor": {

package.json

@@ -49,7 +49,7 @@
     "ws": "^7.3.1"
   },
   "devDependencies": {
-    "@playwright/test-runner": "^0.3.17",
+    "@playwright/test-runner": "^0.3.18",
     "@types/debug": "^4.1.5",
     "@types/extract-zip": "^1.6.2",
     "@types/mime": "^2.0.3",

test/browsertype-launch-server.spec.ts

@@ -26,9 +26,9 @@ describe('lauch server', suite => {
     await browserServer.close();
   });
 
-  it('should work with port', async ({browserType, defaultBrowserOptions, parallelIndex}) => {
-    const browserServer = await browserType.launchServer({ ...defaultBrowserOptions, port: 8800 + parallelIndex });
-    expect(browserServer.wsEndpoint()).toContain(String(8800 + parallelIndex));
+  it('should work with port', async ({browserType, defaultBrowserOptions, workerIndex}) => {
+    const browserServer = await browserType.launchServer({ ...defaultBrowserOptions, port: 8800 + workerIndex });
+    expect(browserServer.wsEndpoint()).toContain(String(8800 + workerIndex));
     await browserServer.close();
   });
 

test/playwright.fixtures.ts

@@ -97,11 +97,11 @@ export const options = {
   TRACING: valueFromEnv('TRACING', false),
 };
 
-defineWorkerFixture('httpService', async ({parallelIndex}, test) => {
+defineWorkerFixture('httpService', async ({workerIndex}, test) => {
   const assetsPath = path.join(__dirname, 'assets');
   const cachedPath = path.join(__dirname, 'assets', 'cached');
 
-  const port = 8907 + parallelIndex * 2;
+  const port = 8907 + workerIndex * 2;
   const server = await TestServer.create(assetsPath, port);
   server.enableHTTPCache(cachedPath);
 
@@ -139,7 +139,7 @@ defineWorkerFixture('defaultBrowserOptions', async ({browserName}, runTest, conf
   });
 });
 
-defineWorkerFixture('playwright', async ({browserName, parallelIndex, platform}, test) => {
+defineWorkerFixture('playwright', async ({browserName, workerIndex, platform}, test) => {
   assert(platform); // Depend on platform to generate all tests.
   const {coverage, uninstall} = installCoverageHooks(browserName);
   if (options.WIRE) {
@@ -172,7 +172,7 @@ defineWorkerFixture('playwright', async ({browserName, parallelIndex, platform},
 
   async function teardownCoverage() {
     uninstall();
-    const coveragePath = path.join(__dirname, 'coverage-report', parallelIndex + '.json');
+    const coveragePath = path.join(__dirname, 'coverage-report', workerIndex + '.json');
     const coverageJSON = [...coverage.keys()].filter(key => coverage.get(key));
     await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true });
     await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8');
@@ -260,10 +260,9 @@ defineWorkerFixture('expectedSSLError', async ({browserName, platform}, runTest)
 });
 
 defineTestFixture('testOutputDir', async ({}, runTest, info) => {
-  const { spec, config } = info;
-  const relativePath = path.relative(config.testDir, spec.file).replace(/\.spec\.[jt]s/, '');
-  const sanitizedTitle = spec.title.replace(/[^\w\d]+/g, '_');
-  const testOutputDir = path.join(config.outputDir, relativePath, sanitizedTitle);
+  const relativePath = path.relative(info.config.testDir, info.file).replace(/\.spec\.[jt]s/, '');
+  const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_');
+  const testOutputDir = path.join(info.config.outputDir, relativePath, sanitizedTitle);
   await fs.promises.mkdir(testOutputDir, { recursive: true });
   await runTest(testOutputDir);
   const files = await fs.promises.readdir(testOutputDir);
@@ -291,8 +290,7 @@ defineTestFixture('context', async ({browser, testOutputDir}, runTest, info) =>
 defineTestFixture('page', async ({context, testOutputDir}, runTest, info) => {
   const page = await context.newPage();
   await runTest(page);
-  const { testRun } = info;
-  if (testRun.status === 'failed' || testRun.status === 'timedOut')
+  if (info.status === 'failed' || info.status === 'timedOut')
     await page.screenshot({ timeout: 5000, path: path.join(testOutputDir, 'test-failed.png') });
 });
 

test/proxy.spec.ts

@@ -121,7 +121,7 @@ it('should exclude patterns', (test, parameters) => {
 
 it('should use socks proxy', (test, parameters) => {
   test.flaky(options.MAC(parameters) && options.WEBKIT(parameters), 'Intermittent page.goto: The network connection was lost error on bots');
-}, async ({ browserType, defaultBrowserOptions, parallelIndex }) => {
+}, async ({ browserType, defaultBrowserOptions, workerIndex }) => {
   const server = socks.createServer((info, accept, deny) => {
     let socket;
     if ((socket = accept(true))) {
@@ -138,7 +138,7 @@ it('should use socks proxy', (test, parameters) => {
       ].join('\r\n'));
     }
   });
-  const socksPort = 9107 + parallelIndex * 2;
+  const socksPort = 9107 + workerIndex * 2;
   server.listen(socksPort, 'localhost');
   server.useAuth(socks.auth.None());
 

test/screencast.spec.ts

@@ -58,8 +58,7 @@ defineTestFixture('videoPlayer', async ({videoPlayerBrowser, server}, test) => {
 });
 
 defineTestFixture('relativeArtifactsPath', async ({browserType}, runTest, info) => {
-  const { spec } = info;
-  const sanitizedTitle = spec.title.replace(/[^\w\d]+/g, '_');
+  const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_');
   const relativeArtifactsPath = `${browserType.name()}-${sanitizedTitle}`;
   await runTest(relativeArtifactsPath);
 });