chore: introduce docker integration dogfood (#9998)

This patch:
- rolls stable-test-runner to Nov 2, 2021 tip-of-tree
- introduces a new npm script, `npm run vtest`, to run Visual Regression
  Tests for our HTML reporter

.github/workflows/tests_primary.yml

@@ -71,3 +71,25 @@ jobs:
       if: matrix.os != 'ubuntu-latest'
     - run: xvfb-run npm run ttest
       if: matrix.os == 'ubuntu-latest'
+
+  test_runner_vrt:
+    name: Test Runner Visual Regression Testing
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: 12
+    - run: npm i -g npm@7
+    - run: npm ci
+      env:
+        DEBUG: pw:install
+    - run: npm run build
+    - run: npx playwright install --with-deps
+    - run: cd tests/playwright-test/stable-test-runner && npx playwright install docker-image
+    - run: npm run vtest
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: visual-regression-testing
+        path: test-results

package.json

@@ -19,6 +19,7 @@
     "atest": "playwright test --config=tests/config/android.config.ts",
     "etest": "playwright test --config=tests/config/electron.config.ts",
     "ttest": "node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright-test.config.ts",
+    "vtest": "cross-env PLAYWRIGHT_DOCKER=1 node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright-test.config.ts",
     "test": "playwright test --config=tests/config/default.config.ts",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext ts,tsx . || eslint --ext ts,tsx .",
     "tsc": "tsc -p .",

tests/playwright-test/playwright-test-fixtures.ts

@@ -121,6 +121,8 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
       PWTEST_CACHE_DIR: cacheDir,
       PWTEST_SKIP_TEST_OUTPUT: '1',
       ...env,
+      PLAYWRIGHT_DOCKER: undefined,
+      PW_GRID: undefined,
     },
     cwd: baseDir,
   });

tests/playwright-test/playwright-test.config.ts

@@ -22,9 +22,11 @@ const config: Config = {
   timeout: 30000,
   forbidOnly: !!process.env.CI,
   preserveOutput: process.env.CI ? 'failures-only' : 'always',
-  projects: [
+  projects: process.env.PLAYWRIGHT_DOCKER ? [
-    { name: 'playwright-test' },
+    { name: 'visual tests', testMatch: ['*.visual.ts'] },
-  ]
+  ] : [
+    { name: 'playwright-test', testIgnore: ['*.visual.ts'] },
+  ],
 };
 
 export default config;

tests/playwright-test/reporter-html.visual.ts

@@ -31,8 +31,6 @@ const test = baseTest.extend<{ showReport: () => Promise<void> }>({
   }
 });
 
-test.use({ channel: 'chrome' });
-
 test('should generate report', async ({ runInlineTest, showReport, page }) => {
   await runInlineTest({
     'playwright.config.ts': `

tests/playwright-test/stable-test-runner/package-lock.json

@@ -5,7 +5,7 @@
   "packages": {
     "": {
       "dependencies": {
-        "@playwright/test": "=1.16.2"
+        "@playwright/test": "^1.17.0-next-1635886706000"
       }
     },
     "node_modules/@babel/code-frame": {
@@ -677,9 +677,9 @@
       }
     },
     "node_modules/@playwright/test": {
-      "version": "1.16.2",
+      "version": "1.17.0-next-1635886706000",
-      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.16.2.tgz",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.17.0-next-1635886706000.tgz",
-      "integrity": "sha512-nZVSGZ3XZxsFKt5TRYUgePsMLDBS8p2E6DPDd/dpICVxzPhBowsBd9IKSlq91faH4BE6nYMDjAfVAmqXLjohkA==",
+      "integrity": "sha512-shbXRTr1af9YuekWz/OAJl/3h2j8uERqzWGjMCTGzA16juBfhhAuzfc+CgRPHbEdKDaKykFmjmeeror02OvjdQ==",
       "dependencies": {
         "@babel/code-frame": "^7.14.5",
         "@babel/core": "^7.14.8",
@@ -709,7 +709,7 @@
         "open": "^8.3.0",
         "pirates": "^4.0.1",
         "pixelmatch": "^5.2.1",
-        "playwright-core": "=1.16.2",
+        "playwright-core": "=1.17.0-next-1635886706000",
         "pngjs": "^5.0.0",
         "rimraf": "^3.0.2",
         "source-map-support": "^0.4.18",
@@ -1543,9 +1543,9 @@
       }
     },
     "node_modules/mime": {
-      "version": "2.5.2",
+      "version": "2.6.0",
-      "resolved": "https://registry.npmjs.org/mime/-/mime-2.5.2.tgz",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz",
-      "integrity": "sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg==",
+      "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==",
       "bin": {
         "mime": "cli.js"
       },
@@ -1688,9 +1688,9 @@
       }
     },
     "node_modules/playwright-core": {
-      "version": "1.16.2",
+      "version": "1.17.0-next-1635886706000",
-      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.16.2.tgz",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.17.0-next-1635886706000.tgz",
-      "integrity": "sha512-8WkoP5OfZAYrRxtW/PCVACn9bNgqrTxVVPlc+MoxvJ48knNsZ+skrPjfno/XF3SgTUY9DyYX0g5fVOB7lkPtGg==",
+      "integrity": "sha512-QenETXX7I7J+s5RYRtZIyW6pKKyfBYNf9XPq/exmv8jBPhhRD+0hy/NB1WwmWUt/nze8GV/ZS+2G5IN2PUXFVw==",
       "dependencies": {
         "commander": "^8.2.0",
         "debug": "^4.1.1",
@@ -2499,9 +2499,9 @@
       }
     },
     "@playwright/test": {
-      "version": "1.16.2",
+      "version": "1.17.0-next-1635886706000",
-      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.16.2.tgz",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.17.0-next-1635886706000.tgz",
-      "integrity": "sha512-nZVSGZ3XZxsFKt5TRYUgePsMLDBS8p2E6DPDd/dpICVxzPhBowsBd9IKSlq91faH4BE6nYMDjAfVAmqXLjohkA==",
+      "integrity": "sha512-shbXRTr1af9YuekWz/OAJl/3h2j8uERqzWGjMCTGzA16juBfhhAuzfc+CgRPHbEdKDaKykFmjmeeror02OvjdQ==",
       "requires": {
         "@babel/code-frame": "^7.14.5",
         "@babel/core": "^7.14.8",
@@ -2531,7 +2531,7 @@
         "open": "^8.3.0",
         "pirates": "^4.0.1",
         "pixelmatch": "^5.2.1",
-        "playwright-core": "=1.16.2",
+        "playwright-core": "=1.17.0-next-1635886706000",
         "pngjs": "^5.0.0",
         "rimraf": "^3.0.2",
         "source-map-support": "^0.4.18",
@@ -3163,9 +3163,9 @@
       }
     },
     "mime": {
-      "version": "2.5.2",
+      "version": "2.6.0",
-      "resolved": "https://registry.npmjs.org/mime/-/mime-2.5.2.tgz",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz",
-      "integrity": "sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg=="
+      "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg=="
     },
     "minimatch": {
       "version": "3.0.4",
@@ -3268,9 +3268,9 @@
       }
     },
     "playwright-core": {
-      "version": "1.16.2",
+      "version": "1.17.0-next-1635886706000",
-      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.16.2.tgz",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.17.0-next-1635886706000.tgz",
-      "integrity": "sha512-8WkoP5OfZAYrRxtW/PCVACn9bNgqrTxVVPlc+MoxvJ48knNsZ+skrPjfno/XF3SgTUY9DyYX0g5fVOB7lkPtGg==",
+      "integrity": "sha512-QenETXX7I7J+s5RYRtZIyW6pKKyfBYNf9XPq/exmv8jBPhhRD+0hy/NB1WwmWUt/nze8GV/ZS+2G5IN2PUXFVw==",
       "requires": {
         "commander": "^8.2.0",
         "debug": "^4.1.1",

tests/playwright-test/stable-test-runner/package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
   "dependencies": {
-    "@playwright/test": "=1.16.2"
+    "@playwright/test": "^1.17.0-next-1635886706000"
   }
 }