diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index 6f88ed77fe..f23adc81b6 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -71,3 +71,25 @@ jobs: if: matrix.os != 'ubuntu-latest' - run: xvfb-run npm run ttest if: matrix.os == 'ubuntu-latest' + + test_runner_vrt: + name: Test Runner Visual Regression Testing + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: 12 + - run: npm i -g npm@7 + - run: npm ci + env: + DEBUG: pw:install + - run: npm run build + - run: npx playwright install --with-deps + - run: cd tests/playwright-test/stable-test-runner && npx playwright install docker-image + - run: npm run vtest + - uses: actions/upload-artifact@v1 + if: ${{ always() }} + with: + name: visual-regression-testing + path: test-results diff --git a/package.json b/package.json index a0944134c6..0d9a632afa 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "atest": "playwright test --config=tests/config/android.config.ts", "etest": "playwright test --config=tests/config/electron.config.ts", "ttest": "node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright-test.config.ts", + "vtest": "cross-env PLAYWRIGHT_DOCKER=1 node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright-test.config.ts", "test": "playwright test --config=tests/config/default.config.ts", "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext ts,tsx . || eslint --ext ts,tsx .", "tsc": "tsc -p .", diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index 8f4fda7e28..420475742a 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -121,6 +121,8 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b PWTEST_CACHE_DIR: cacheDir, PWTEST_SKIP_TEST_OUTPUT: '1', ...env, + PLAYWRIGHT_DOCKER: undefined, + PW_GRID: undefined, }, cwd: baseDir, }); diff --git a/tests/playwright-test/playwright-test.config.ts b/tests/playwright-test/playwright-test.config.ts index d88804257b..6056acdd01 100644 --- a/tests/playwright-test/playwright-test.config.ts +++ b/tests/playwright-test/playwright-test.config.ts @@ -22,9 +22,11 @@ const config: Config = { timeout: 30000, forbidOnly: !!process.env.CI, preserveOutput: process.env.CI ? 'failures-only' : 'always', - projects: [ - { name: 'playwright-test' }, - ] + projects: process.env.PLAYWRIGHT_DOCKER ? [ + { name: 'visual tests', testMatch: ['*.visual.ts'] }, + ] : [ + { name: 'playwright-test', testIgnore: ['*.visual.ts'] }, + ], }; export default config; diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.visual.ts similarity index 99% rename from tests/playwright-test/reporter-html.spec.ts rename to tests/playwright-test/reporter-html.visual.ts index ddc71cc0cf..175443428b 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.visual.ts @@ -31,8 +31,6 @@ const test = baseTest.extend<{ showReport: () => Promise }>({ } }); -test.use({ channel: 'chrome' }); - test('should generate report', async ({ runInlineTest, showReport, page }) => { await runInlineTest({ 'playwright.config.ts': ` diff --git a/tests/playwright-test/stable-test-runner/package-lock.json b/tests/playwright-test/stable-test-runner/package-lock.json index 6abdd0f994..a32f5de875 100644 --- a/tests/playwright-test/stable-test-runner/package-lock.json +++ b/tests/playwright-test/stable-test-runner/package-lock.json @@ -5,7 +5,7 @@ "packages": { "": { "dependencies": { - "@playwright/test": "=1.16.2" + "@playwright/test": "^1.17.0-next-1635886706000" } }, "node_modules/@babel/code-frame": { @@ -677,9 +677,9 @@ } }, "node_modules/@playwright/test": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.16.2.tgz", - "integrity": "sha512-nZVSGZ3XZxsFKt5TRYUgePsMLDBS8p2E6DPDd/dpICVxzPhBowsBd9IKSlq91faH4BE6nYMDjAfVAmqXLjohkA==", + "version": "1.17.0-next-1635886706000", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.17.0-next-1635886706000.tgz", + "integrity": "sha512-shbXRTr1af9YuekWz/OAJl/3h2j8uERqzWGjMCTGzA16juBfhhAuzfc+CgRPHbEdKDaKykFmjmeeror02OvjdQ==", "dependencies": { "@babel/code-frame": "^7.14.5", "@babel/core": "^7.14.8", @@ -709,7 +709,7 @@ "open": "^8.3.0", "pirates": "^4.0.1", "pixelmatch": "^5.2.1", - "playwright-core": "=1.16.2", + "playwright-core": "=1.17.0-next-1635886706000", "pngjs": "^5.0.0", "rimraf": "^3.0.2", "source-map-support": "^0.4.18", @@ -1543,9 +1543,9 @@ } }, "node_modules/mime": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.5.2.tgz", - "integrity": "sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz", + "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==", "bin": { "mime": "cli.js" }, @@ -1688,9 +1688,9 @@ } }, "node_modules/playwright-core": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.16.2.tgz", - "integrity": "sha512-8WkoP5OfZAYrRxtW/PCVACn9bNgqrTxVVPlc+MoxvJ48knNsZ+skrPjfno/XF3SgTUY9DyYX0g5fVOB7lkPtGg==", + "version": "1.17.0-next-1635886706000", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.17.0-next-1635886706000.tgz", + "integrity": "sha512-QenETXX7I7J+s5RYRtZIyW6pKKyfBYNf9XPq/exmv8jBPhhRD+0hy/NB1WwmWUt/nze8GV/ZS+2G5IN2PUXFVw==", "dependencies": { "commander": "^8.2.0", "debug": "^4.1.1", @@ -2499,9 +2499,9 @@ } }, "@playwright/test": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.16.2.tgz", - "integrity": "sha512-nZVSGZ3XZxsFKt5TRYUgePsMLDBS8p2E6DPDd/dpICVxzPhBowsBd9IKSlq91faH4BE6nYMDjAfVAmqXLjohkA==", + "version": "1.17.0-next-1635886706000", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.17.0-next-1635886706000.tgz", + "integrity": "sha512-shbXRTr1af9YuekWz/OAJl/3h2j8uERqzWGjMCTGzA16juBfhhAuzfc+CgRPHbEdKDaKykFmjmeeror02OvjdQ==", "requires": { "@babel/code-frame": "^7.14.5", "@babel/core": "^7.14.8", @@ -2531,7 +2531,7 @@ "open": "^8.3.0", "pirates": "^4.0.1", "pixelmatch": "^5.2.1", - "playwright-core": "=1.16.2", + "playwright-core": "=1.17.0-next-1635886706000", "pngjs": "^5.0.0", "rimraf": "^3.0.2", "source-map-support": "^0.4.18", @@ -3163,9 +3163,9 @@ } }, "mime": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.5.2.tgz", - "integrity": "sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg==" + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz", + "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==" }, "minimatch": { "version": "3.0.4", @@ -3268,9 +3268,9 @@ } }, "playwright-core": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.16.2.tgz", - "integrity": "sha512-8WkoP5OfZAYrRxtW/PCVACn9bNgqrTxVVPlc+MoxvJ48knNsZ+skrPjfno/XF3SgTUY9DyYX0g5fVOB7lkPtGg==", + "version": "1.17.0-next-1635886706000", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.17.0-next-1635886706000.tgz", + "integrity": "sha512-QenETXX7I7J+s5RYRtZIyW6pKKyfBYNf9XPq/exmv8jBPhhRD+0hy/NB1WwmWUt/nze8GV/ZS+2G5IN2PUXFVw==", "requires": { "commander": "^8.2.0", "debug": "^4.1.1", diff --git a/tests/playwright-test/stable-test-runner/package.json b/tests/playwright-test/stable-test-runner/package.json index 41eacc4bd6..f47ae4f6d6 100644 --- a/tests/playwright-test/stable-test-runner/package.json +++ b/tests/playwright-test/stable-test-runner/package.json @@ -1,6 +1,6 @@ { "private": true, "dependencies": { - "@playwright/test": "=1.16.2" + "@playwright/test": "^1.17.0-next-1635886706000" } }