diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e6279df909..1b6c40a509 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,8 +46,8 @@ jobs: - uses: actions/upload-artifact@v1 if: failure() with: - name: ${{ matrix.browser }}-${{ matrix.os }}-output - path: test/output-${{ matrix.browser }} + name: ${{ matrix.browser }}-${{ matrix.os }}-test-results + path: test-results - uses: actions/upload-artifact@v1 if: ${{ always() }} with: @@ -77,8 +77,8 @@ jobs: - uses: actions/upload-artifact@v1 if: failure() with: - name: ${{ matrix.browser }}-mac-output - path: test/output-${{ matrix.browser }} + name: ${{ matrix.browser }}-mac-test-results + path: test-results - uses: actions/upload-artifact@v1 if: ${{ always() }} with: @@ -112,8 +112,8 @@ jobs: - uses: actions/upload-artifact@v1 if: failure() with: - name: ${{ matrix.browser }}-win-output - path: test/output-${{ matrix.browser }} + name: ${{ matrix.browser }}-win-test-results + path: test-results - uses: actions/upload-artifact@v1 if: ${{ always() }} with: @@ -169,8 +169,8 @@ jobs: - uses: actions/upload-artifact@v1 if: failure() with: - name: headful-${{ matrix.browser }}-linux-output - path: test/output-${{ matrix.browser }} + name: headful-${{ matrix.browser }}-linux-test-results + path: test-results - uses: actions/upload-artifact@v1 if: ${{ always() }} with: @@ -207,8 +207,8 @@ jobs: - uses: actions/upload-artifact@v1 if: failure() with: - name: wire-${{ matrix.browser }}-linux-output - path: test/output-${{ matrix.browser }} + name: wire-${{ matrix.browser }}-linux-test-results + path: test-results - uses: actions/upload-artifact@v1 if: ${{ always() }} with: diff --git a/.gitignore b/.gitignore index 92cc44e8a6..657282ae6f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ /node_modules/ -/test/output-chromium -/test/output-firefox -/test/output-webkit +/test-results/ +/test/coverage-report /test/test-user-data-dir* .local-browsers/ /.dev_profile* diff --git a/test/__snapshots__/chromium/oopif/chromium/screenshot-oopif.png b/test/__snapshots__/chromium/oopif/chromium/screenshot-oopif.png new file mode 100644 index 0000000000..951e96f178 Binary files /dev/null and b/test/__snapshots__/chromium/oopif/chromium/screenshot-oopif.png differ diff --git a/test/golden-chromium/screenshot-element-bounding-box.png b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-bounding-box.png similarity index 100% rename from test/golden-chromium/screenshot-element-bounding-box.png rename to test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-bounding-box.png diff --git a/test/golden-chromium/screenshot-element-fractional-offset.png b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-fractional-offset.png similarity index 100% rename from test/golden-chromium/screenshot-element-fractional-offset.png rename to test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-fractional-offset.png diff --git a/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-fractional.png b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-fractional.png new file mode 100644 index 0000000000..bd1810da8b Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-fractional.png differ diff --git a/test/golden-chromium/screenshot-element-larger-than-viewport.png b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-larger-than-viewport.png similarity index 100% rename from test/golden-chromium/screenshot-element-larger-than-viewport.png rename to test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-larger-than-viewport.png diff --git a/test/golden-chromium/screenshot-element-mobile-dsf.png b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-mobile-dsf.png similarity index 100% rename from test/golden-chromium/screenshot-element-mobile-dsf.png rename to test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-mobile-dsf.png diff --git a/test/golden-chromium/screenshot-element-mobile.png b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-mobile.png similarity index 100% rename from test/golden-chromium/screenshot-element-mobile.png rename to test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-mobile.png diff --git a/test/golden-chromium/screenshot-element-padding-border.png b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-padding-border.png similarity index 100% rename from test/golden-chromium/screenshot-element-padding-border.png rename to test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-padding-border.png diff --git a/test/golden-chromium/screenshot-element-rotate.png b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-rotate.png similarity index 100% rename from test/golden-chromium/screenshot-element-rotate.png rename to test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-rotate.png diff --git a/test/golden-chromium/screenshot-element-scrolled-into-view.png b/test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-scrolled-into-view.png similarity index 100% rename from test/golden-chromium/screenshot-element-scrolled-into-view.png rename to test/__snapshots__/elementhandle-screenshot/chromium/screenshot-element-scrolled-into-view.png diff --git a/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-bounding-box.png b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-bounding-box.png new file mode 100644 index 0000000000..9e208f86d8 Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-bounding-box.png differ diff --git a/test/golden-firefox/screenshot-element-fractional-offset.png b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-fractional-offset.png similarity index 100% rename from test/golden-firefox/screenshot-element-fractional-offset.png rename to test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-fractional-offset.png diff --git a/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-fractional.png b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-fractional.png new file mode 100644 index 0000000000..d1431bd91d Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-fractional.png differ diff --git a/test/golden-webkit/screenshot-element-larger-than-viewport.png b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-larger-than-viewport.png similarity index 85% rename from test/golden-webkit/screenshot-element-larger-than-viewport.png rename to test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-larger-than-viewport.png index 5fcdb92355..6d28cddcea 100644 Binary files a/test/golden-webkit/screenshot-element-larger-than-viewport.png and b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-larger-than-viewport.png differ diff --git a/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-padding-border.png b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-padding-border.png new file mode 100644 index 0000000000..2b72c7528b Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-padding-border.png differ diff --git a/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-rotate.png b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-rotate.png new file mode 100644 index 0000000000..f391edcad9 Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-rotate.png differ diff --git a/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-scrolled-into-view.png b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-scrolled-into-view.png new file mode 100644 index 0000000000..2b72c7528b Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/firefox/screenshot-element-scrolled-into-view.png differ diff --git a/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-bounding-box.png b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-bounding-box.png new file mode 100644 index 0000000000..ad6858bf2f Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-bounding-box.png differ diff --git a/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-fractional-offset.png b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-fractional-offset.png new file mode 100644 index 0000000000..ac79d2f938 Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-fractional-offset.png differ diff --git a/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-fractional.png b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-fractional.png new file mode 100644 index 0000000000..03c539bc86 Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-fractional.png differ diff --git a/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-larger-than-viewport.png b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-larger-than-viewport.png new file mode 100644 index 0000000000..aacf91548e Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-larger-than-viewport.png differ diff --git a/test/golden-webkit/screenshot-element-mobile-dsf.png b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-mobile-dsf.png similarity index 100% rename from test/golden-webkit/screenshot-element-mobile-dsf.png rename to test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-mobile-dsf.png diff --git a/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-mobile.png b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-mobile.png new file mode 100644 index 0000000000..ad6858bf2f Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-mobile.png differ diff --git a/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-padding-border.png b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-padding-border.png new file mode 100644 index 0000000000..4a1e362cdb Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-padding-border.png differ diff --git a/test/golden-firefox/screenshot-element-rotate.png b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-rotate.png similarity index 100% rename from test/golden-firefox/screenshot-element-rotate.png rename to test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-rotate.png diff --git a/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-scrolled-into-view.png b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-scrolled-into-view.png new file mode 100644 index 0000000000..4a1e362cdb Binary files /dev/null and b/test/__snapshots__/elementhandle-screenshot/webkit/screenshot-element-scrolled-into-view.png differ diff --git a/test/__snapshots__/emulation-focus/chromium/grid-cell-0.png b/test/__snapshots__/emulation-focus/chromium/grid-cell-0.png new file mode 100644 index 0000000000..ca5f8ec584 Binary files /dev/null and b/test/__snapshots__/emulation-focus/chromium/grid-cell-0.png differ diff --git a/test/golden-chromium/screenshot-sanity.png b/test/__snapshots__/emulation-focus/chromium/screenshot-sanity.png similarity index 100% rename from test/golden-chromium/screenshot-sanity.png rename to test/__snapshots__/emulation-focus/chromium/screenshot-sanity.png diff --git a/test/__snapshots__/emulation-focus/firefox/grid-cell-0.png b/test/__snapshots__/emulation-focus/firefox/grid-cell-0.png new file mode 100644 index 0000000000..370350757a Binary files /dev/null and b/test/__snapshots__/emulation-focus/firefox/grid-cell-0.png differ diff --git a/test/__snapshots__/emulation-focus/firefox/screenshot-sanity.png b/test/__snapshots__/emulation-focus/firefox/screenshot-sanity.png new file mode 100644 index 0000000000..7b16493355 Binary files /dev/null and b/test/__snapshots__/emulation-focus/firefox/screenshot-sanity.png differ diff --git a/test/__snapshots__/emulation-focus/webkit/grid-cell-0.png b/test/__snapshots__/emulation-focus/webkit/grid-cell-0.png new file mode 100644 index 0000000000..117929e479 Binary files /dev/null and b/test/__snapshots__/emulation-focus/webkit/grid-cell-0.png differ diff --git a/test/__snapshots__/emulation-focus/webkit/screenshot-sanity.png b/test/__snapshots__/emulation-focus/webkit/screenshot-sanity.png new file mode 100644 index 0000000000..4cf1a338fc Binary files /dev/null and b/test/__snapshots__/emulation-focus/webkit/screenshot-sanity.png differ diff --git a/test/__snapshots__/page-screenshot/chromium/grid-cell-0.png b/test/__snapshots__/page-screenshot/chromium/grid-cell-0.png new file mode 100644 index 0000000000..ca5f8ec584 Binary files /dev/null and b/test/__snapshots__/page-screenshot/chromium/grid-cell-0.png differ diff --git a/test/golden-chromium/grid-cell-1.png b/test/__snapshots__/page-screenshot/chromium/grid-cell-1.png similarity index 100% rename from test/golden-chromium/grid-cell-1.png rename to test/__snapshots__/page-screenshot/chromium/grid-cell-1.png diff --git a/test/golden-chromium/screenshot-canvas.png b/test/__snapshots__/page-screenshot/chromium/screenshot-canvas.png similarity index 100% rename from test/golden-chromium/screenshot-canvas.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-canvas.png diff --git a/test/golden-chromium/screenshot-clip-odd-size.png b/test/__snapshots__/page-screenshot/chromium/screenshot-clip-odd-size.png similarity index 100% rename from test/golden-chromium/screenshot-clip-odd-size.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-clip-odd-size.png diff --git a/test/golden-chromium/screenshot-clip-rect.png b/test/__snapshots__/page-screenshot/chromium/screenshot-clip-rect.png similarity index 100% rename from test/golden-chromium/screenshot-clip-rect.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-clip-rect.png diff --git a/test/golden-chromium/screenshot-device-scale-factor.png b/test/__snapshots__/page-screenshot/chromium/screenshot-device-scale-factor.png similarity index 100% rename from test/golden-chromium/screenshot-device-scale-factor.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-device-scale-factor.png diff --git a/test/golden-chromium/screenshot-grid-fullpage.png b/test/__snapshots__/page-screenshot/chromium/screenshot-grid-fullpage.png similarity index 100% rename from test/golden-chromium/screenshot-grid-fullpage.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-grid-fullpage.png diff --git a/test/golden-chromium/screenshot-iframe.png b/test/__snapshots__/page-screenshot/chromium/screenshot-iframe.png similarity index 100% rename from test/golden-chromium/screenshot-iframe.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-iframe.png diff --git a/test/golden-chromium/screenshot-mobile-clip.png b/test/__snapshots__/page-screenshot/chromium/screenshot-mobile-clip.png similarity index 100% rename from test/golden-chromium/screenshot-mobile-clip.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-mobile-clip.png diff --git a/test/golden-chromium/screenshot-mobile-fullpage.png b/test/__snapshots__/page-screenshot/chromium/screenshot-mobile-fullpage.png similarity index 100% rename from test/golden-chromium/screenshot-mobile-fullpage.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-mobile-fullpage.png diff --git a/test/__snapshots__/page-screenshot/chromium/screenshot-mobile.png b/test/__snapshots__/page-screenshot/chromium/screenshot-mobile.png new file mode 100644 index 0000000000..f18ddb47f3 Binary files /dev/null and b/test/__snapshots__/page-screenshot/chromium/screenshot-mobile.png differ diff --git a/test/golden-chromium/screenshot-offscreen-clip.png b/test/__snapshots__/page-screenshot/chromium/screenshot-offscreen-clip.png similarity index 100% rename from test/golden-chromium/screenshot-offscreen-clip.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-offscreen-clip.png diff --git a/test/__snapshots__/page-screenshot/chromium/screenshot-sanity.png b/test/__snapshots__/page-screenshot/chromium/screenshot-sanity.png new file mode 100644 index 0000000000..122a4f0ae0 Binary files /dev/null and b/test/__snapshots__/page-screenshot/chromium/screenshot-sanity.png differ diff --git a/test/golden-chromium/screenshot-translateZ.png b/test/__snapshots__/page-screenshot/chromium/screenshot-translateZ.png similarity index 100% rename from test/golden-chromium/screenshot-translateZ.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-translateZ.png diff --git a/test/golden-chromium/screenshot-webgl.png b/test/__snapshots__/page-screenshot/chromium/screenshot-webgl.png similarity index 100% rename from test/golden-chromium/screenshot-webgl.png rename to test/__snapshots__/page-screenshot/chromium/screenshot-webgl.png diff --git a/test/golden-chromium/transparent.png b/test/__snapshots__/page-screenshot/chromium/transparent.png similarity index 100% rename from test/golden-chromium/transparent.png rename to test/__snapshots__/page-screenshot/chromium/transparent.png diff --git a/test/__snapshots__/page-screenshot/chromium/white.jpg b/test/__snapshots__/page-screenshot/chromium/white.jpg new file mode 100644 index 0000000000..0bc91643aa Binary files /dev/null and b/test/__snapshots__/page-screenshot/chromium/white.jpg differ diff --git a/test/__snapshots__/page-screenshot/firefox/grid-cell-0.png b/test/__snapshots__/page-screenshot/firefox/grid-cell-0.png new file mode 100644 index 0000000000..370350757a Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/grid-cell-0.png differ diff --git a/test/__snapshots__/page-screenshot/firefox/grid-cell-1.png b/test/__snapshots__/page-screenshot/firefox/grid-cell-1.png new file mode 100644 index 0000000000..55d6a44412 Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/grid-cell-1.png differ diff --git a/test/golden-firefox/screenshot-canvas.png b/test/__snapshots__/page-screenshot/firefox/screenshot-canvas.png similarity index 50% rename from test/golden-firefox/screenshot-canvas.png rename to test/__snapshots__/page-screenshot/firefox/screenshot-canvas.png index dfafbda638..b0af90ae99 100644 Binary files a/test/golden-firefox/screenshot-canvas.png and b/test/__snapshots__/page-screenshot/firefox/screenshot-canvas.png differ diff --git a/test/__snapshots__/page-screenshot/firefox/screenshot-clip-odd-size.png b/test/__snapshots__/page-screenshot/firefox/screenshot-clip-odd-size.png new file mode 100644 index 0000000000..8e86dc9017 Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/screenshot-clip-odd-size.png differ diff --git a/test/__snapshots__/page-screenshot/firefox/screenshot-clip-rect.png b/test/__snapshots__/page-screenshot/firefox/screenshot-clip-rect.png new file mode 100644 index 0000000000..6459e3afe7 Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/screenshot-clip-rect.png differ diff --git a/test/golden-firefox/screenshot-device-scale-factor.png b/test/__snapshots__/page-screenshot/firefox/screenshot-device-scale-factor.png similarity index 100% rename from test/golden-firefox/screenshot-device-scale-factor.png rename to test/__snapshots__/page-screenshot/firefox/screenshot-device-scale-factor.png diff --git a/test/__snapshots__/page-screenshot/firefox/screenshot-grid-fullpage.png b/test/__snapshots__/page-screenshot/firefox/screenshot-grid-fullpage.png new file mode 100644 index 0000000000..e87220e057 Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/screenshot-grid-fullpage.png differ diff --git a/test/golden-firefox/screenshot-iframe.png b/test/__snapshots__/page-screenshot/firefox/screenshot-iframe.png similarity index 100% rename from test/golden-firefox/screenshot-iframe.png rename to test/__snapshots__/page-screenshot/firefox/screenshot-iframe.png diff --git a/test/__snapshots__/page-screenshot/firefox/screenshot-offscreen-clip.png b/test/__snapshots__/page-screenshot/firefox/screenshot-offscreen-clip.png new file mode 100644 index 0000000000..cfab2376be Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/screenshot-offscreen-clip.png differ diff --git a/test/__snapshots__/page-screenshot/firefox/screenshot-sanity.png b/test/__snapshots__/page-screenshot/firefox/screenshot-sanity.png new file mode 100644 index 0000000000..7b16493355 Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/screenshot-sanity.png differ diff --git a/test/__snapshots__/page-screenshot/firefox/screenshot-translateZ.png b/test/__snapshots__/page-screenshot/firefox/screenshot-translateZ.png new file mode 100644 index 0000000000..7dc10f2f5a Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/screenshot-translateZ.png differ diff --git a/test/__snapshots__/page-screenshot/firefox/white.jpg b/test/__snapshots__/page-screenshot/firefox/white.jpg new file mode 100644 index 0000000000..40eb86d77b Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/white.jpg differ diff --git a/test/__snapshots__/page-screenshot/webkit/grid-cell-0.png b/test/__snapshots__/page-screenshot/webkit/grid-cell-0.png new file mode 100644 index 0000000000..117929e479 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/grid-cell-0.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/grid-cell-1.png b/test/__snapshots__/page-screenshot/webkit/grid-cell-1.png new file mode 100644 index 0000000000..9007ff0a95 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/grid-cell-1.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-canvas.png b/test/__snapshots__/page-screenshot/webkit/screenshot-canvas.png new file mode 100644 index 0000000000..0a40e94959 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-canvas.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-clip-odd-size.png b/test/__snapshots__/page-screenshot/webkit/screenshot-clip-odd-size.png new file mode 100644 index 0000000000..86a4d51ca0 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-clip-odd-size.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-clip-rect.png b/test/__snapshots__/page-screenshot/webkit/screenshot-clip-rect.png new file mode 100644 index 0000000000..8bfbcda983 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-clip-rect.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-device-scale-factor.png b/test/__snapshots__/page-screenshot/webkit/screenshot-device-scale-factor.png new file mode 100644 index 0000000000..a9729736ef Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-device-scale-factor.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-grid-fullpage.png b/test/__snapshots__/page-screenshot/webkit/screenshot-grid-fullpage.png new file mode 100644 index 0000000000..494d216e00 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-grid-fullpage.png differ diff --git a/test/golden-webkit/screenshot-iframe.png b/test/__snapshots__/page-screenshot/webkit/screenshot-iframe.png similarity index 100% rename from test/golden-webkit/screenshot-iframe.png rename to test/__snapshots__/page-screenshot/webkit/screenshot-iframe.png diff --git a/test/golden-webkit/screenshot-mobile-clip.png b/test/__snapshots__/page-screenshot/webkit/screenshot-mobile-clip.png similarity index 100% rename from test/golden-webkit/screenshot-mobile-clip.png rename to test/__snapshots__/page-screenshot/webkit/screenshot-mobile-clip.png diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-mobile-fullpage.png b/test/__snapshots__/page-screenshot/webkit/screenshot-mobile-fullpage.png new file mode 100644 index 0000000000..8e72219561 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-mobile-fullpage.png differ diff --git a/test/golden-chromium/screenshot-mobile.png b/test/__snapshots__/page-screenshot/webkit/screenshot-mobile.png similarity index 100% rename from test/golden-chromium/screenshot-mobile.png rename to test/__snapshots__/page-screenshot/webkit/screenshot-mobile.png diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-offscreen-clip.png b/test/__snapshots__/page-screenshot/webkit/screenshot-offscreen-clip.png new file mode 100644 index 0000000000..9882dd3b49 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-offscreen-clip.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-sanity.png b/test/__snapshots__/page-screenshot/webkit/screenshot-sanity.png new file mode 100644 index 0000000000..4cf1a338fc Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-sanity.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-translateZ.png b/test/__snapshots__/page-screenshot/webkit/screenshot-translateZ.png new file mode 100644 index 0000000000..74a525b9c5 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-translateZ.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/transparent.png b/test/__snapshots__/page-screenshot/webkit/transparent.png new file mode 100644 index 0000000000..2c60e44978 Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/transparent.png differ diff --git a/test/__snapshots__/page-screenshot/webkit/white.jpg b/test/__snapshots__/page-screenshot/webkit/white.jpg new file mode 100644 index 0000000000..3f6faefd6e Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/white.jpg differ diff --git a/test/__snapshots__/request-fulfill/chromium/mock-binary-response.png b/test/__snapshots__/request-fulfill/chromium/mock-binary-response.png new file mode 100644 index 0000000000..e4f6191796 Binary files /dev/null and b/test/__snapshots__/request-fulfill/chromium/mock-binary-response.png differ diff --git a/test/golden-chromium/mock-svg.png b/test/__snapshots__/request-fulfill/chromium/mock-svg.png similarity index 100% rename from test/golden-chromium/mock-svg.png rename to test/__snapshots__/request-fulfill/chromium/mock-svg.png diff --git a/test/golden-firefox/mock-binary-response.png b/test/__snapshots__/request-fulfill/firefox/mock-binary-response.png similarity index 100% rename from test/golden-firefox/mock-binary-response.png rename to test/__snapshots__/request-fulfill/firefox/mock-binary-response.png diff --git a/test/golden-firefox/mock-svg.png b/test/__snapshots__/request-fulfill/firefox/mock-svg.png similarity index 100% rename from test/golden-firefox/mock-svg.png rename to test/__snapshots__/request-fulfill/firefox/mock-svg.png diff --git a/test/__snapshots__/request-fulfill/webkit/mock-binary-response.png b/test/__snapshots__/request-fulfill/webkit/mock-binary-response.png new file mode 100644 index 0000000000..c7598e390e Binary files /dev/null and b/test/__snapshots__/request-fulfill/webkit/mock-binary-response.png differ diff --git a/test/golden-webkit/mock-svg.png b/test/__snapshots__/request-fulfill/webkit/mock-svg.png similarity index 100% rename from test/golden-webkit/mock-svg.png rename to test/__snapshots__/request-fulfill/webkit/mock-svg.png diff --git a/test/base.fixture.ts b/test/base.fixture.ts index c781ea5b04..ed1e75f9bb 100644 --- a/test/base.fixture.ts +++ b/test/base.fixture.ts @@ -26,6 +26,7 @@ import { setUnderTest } from '../lib/helper'; import { installCoverageHooks } from './runner/coverage'; import { valueFromEnv } from './runner/utils'; import { registerFixture, registerWorkerFixture} from './runner/fixtures'; +import './runner/builtin.fixtures'; import {mkdtempAsync, removeFolderAsync} from './utils'; @@ -37,13 +38,11 @@ const platform = os.platform(); declare global { interface WorkerState { asset: (path: string) => string; - parallelIndex: number; defaultBrowserOptions: LaunchOptions; golden: (path: string) => string; playwright: typeof import('../index'); browserType: BrowserType; browser: Browser; - tmpDir: string; } interface FixtureState { toImpl: (rpcObject: any) => any; @@ -62,10 +61,6 @@ declare global { (global as any).FFOX = browserName === 'firefox'; (global as any).WEBKIT = browserName === 'webkit'; -registerWorkerFixture('parallelIndex', async ({}, test) => { - await test(parseInt(process.env.JEST_WORKER_ID, 10) - 1); -}); - registerWorkerFixture('httpService', async ({parallelIndex}, test) => { const assetsPath = path.join(__dirname, 'assets'); const cachedPath = path.join(__dirname, 'assets', 'cached'); @@ -139,7 +134,7 @@ registerWorkerFixture('playwright', async({parallelIndex}, test) => { async function teardownCoverage() { uninstall(); - const coveragePath = path.join(path.join(__dirname, 'output-' + browserName), 'coverage', parallelIndex + '.json'); + const coveragePath = path.join(path.join(__dirname, 'coverage-report'), 'coverage', parallelIndex + '.json'); const coverageJSON = [...coverage.keys()].filter(key => coverage.get(key)); await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true }); await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8'); @@ -205,5 +200,5 @@ registerWorkerFixture('asset', async ({}, test) => { }); registerWorkerFixture('golden', async ({browserName}, test) => { - await test(p => path.join(__dirname, `golden-${browserName}`, p)); + await test(p => path.join(`${browserName}`, p)); }); diff --git a/test/chromium/oopif.spec.ts b/test/chromium/oopif.spec.ts index d275b0c7b8..00ec54ac9c 100644 --- a/test/chromium/oopif.spec.ts +++ b/test/chromium/oopif.spec.ts @@ -155,7 +155,7 @@ it.skip(!CHROMIUM)('should take screenshot', async({browser, page, server, golde await page.goto(server.PREFIX + '/dynamic-oopif.html'); expect(page.frames().length).toBe(2); expect(await countOOPIFs(browser)).toBe(1); - expect(await page.screenshot()).toMatchImage(golden('screenshot-oopif.png')); + expect(await page.screenshot()).toMatchImage(golden('screenshot-oopif.png'), { threshold: 0.3 }); }); it.skip(!CHROMIUM)('should load oopif iframes with subresources and request interception', async function({browser, page, server, context}) { diff --git a/test/chromium/tracing.spec.ts b/test/chromium/tracing.spec.ts index b58701f130..2b8940521f 100644 --- a/test/chromium/tracing.spec.ts +++ b/test/chromium/tracing.spec.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import '../runner/builtin.fixtures'; import '../base.fixture'; import { registerFixture } from '../runner/fixtures'; diff --git a/test/defaultbrowsercontext.spec.ts b/test/defaultbrowsercontext.spec.ts index 3ab31c5d90..b49acf26f9 100644 --- a/test/defaultbrowsercontext.spec.ts +++ b/test/defaultbrowsercontext.spec.ts @@ -14,6 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import './runner/builtin.fixtures'; import './base.fixture'; import { registerFixture } from './runner/fixtures'; import fs from 'fs'; diff --git a/test/download.spec.ts b/test/download.spec.ts index 006692b099..c3c048c9a7 100644 --- a/test/download.spec.ts +++ b/test/download.spec.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import './runner/builtin.fixtures'; import './base.fixture'; import fs from 'fs'; diff --git a/test/downloads-path.spec.ts b/test/downloads-path.spec.ts index 2acfbae624..73031f8a70 100644 --- a/test/downloads-path.spec.ts +++ b/test/downloads-path.spec.ts @@ -13,7 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + + +import './runner/builtin.fixtures'; import './base.fixture'; + import { registerFixture } from './runner/fixtures'; import path from 'path'; diff --git a/test/golden-chromium/csscoverage-involved.txt b/test/golden-chromium/csscoverage-involved.txt deleted file mode 100644 index 9b851d0bd3..0000000000 --- a/test/golden-chromium/csscoverage-involved.txt +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "url": "http://localhost:/csscoverage/involved.html", - "ranges": [ - { - "start": 149, - "end": 297 - }, - { - "start": 327, - "end": 433 - } - ], - "text": "\n@charset \"utf-8\";\n@namespace svg url(http://www.w3.org/2000/svg);\n@font-face {\n font-family: \"Example Font\";\n src: url(\"./Dosis-Regular.ttf\");\n}\n\n#fluffy {\n border: 1px solid black;\n z-index: 1;\n /* -webkit-disabled-property: rgb(1, 2, 3) */\n -lol-cats: \"dogs\" /* non-existing property */\n}\n\n@media (min-width: 1px) {\n span {\n -webkit-border-radius: 10px;\n font-family: \"Example Font\";\n animation: 1s identifier;\n }\n}\n" - } -] \ No newline at end of file diff --git a/test/golden-chromium/grid-cell-0.png b/test/golden-chromium/grid-cell-0.png deleted file mode 100644 index ff282e989b..0000000000 Binary files a/test/golden-chromium/grid-cell-0.png and /dev/null differ diff --git a/test/golden-chromium/grid-cell-2.png b/test/golden-chromium/grid-cell-2.png deleted file mode 100644 index 7b01753b6a..0000000000 Binary files a/test/golden-chromium/grid-cell-2.png and /dev/null differ diff --git a/test/golden-chromium/grid-cell-3.png b/test/golden-chromium/grid-cell-3.png deleted file mode 100644 index b9b8b2922b..0000000000 Binary files a/test/golden-chromium/grid-cell-3.png and /dev/null differ diff --git a/test/golden-chromium/jscoverage-involved.txt b/test/golden-chromium/jscoverage-involved.txt deleted file mode 100644 index 6f28e1580e..0000000000 --- a/test/golden-chromium/jscoverage-involved.txt +++ /dev/null @@ -1,28 +0,0 @@ -[ - { - "url": "http://localhost:/jscoverage/involved.html", - "ranges": [ - { - "start": 0, - "end": 35 - }, - { - "start": 50, - "end": 100 - }, - { - "start": 107, - "end": 141 - }, - { - "start": 148, - "end": 160 - }, - { - "start": 168, - "end": 207 - } - ], - "text": "\nfunction foo() {\n if (1 > 2)\n console.log(1);\n if (1 < 2)\n console.log(2);\n let x = 1 > 2 ? 'foo' : 'bar';\n let y = 1 < 2 ? 'foo' : 'bar';\n let z = () => {};\n let q = () => {};\n q();\n}\n\nfoo();\n" - } -] \ No newline at end of file diff --git a/test/golden-chromium/mock-binary-response.png b/test/golden-chromium/mock-binary-response.png deleted file mode 100644 index 8595e0598e..0000000000 Binary files a/test/golden-chromium/mock-binary-response.png and /dev/null differ diff --git a/test/golden-chromium/screenshot-element-fractional.png b/test/golden-chromium/screenshot-element-fractional.png deleted file mode 100644 index 35c53377f9..0000000000 Binary files a/test/golden-chromium/screenshot-element-fractional.png and /dev/null differ diff --git a/test/golden-chromium/screenshot-oopif.png b/test/golden-chromium/screenshot-oopif.png deleted file mode 100644 index 56aa8706bb..0000000000 Binary files a/test/golden-chromium/screenshot-oopif.png and /dev/null differ diff --git a/test/golden-chromium/white.jpg b/test/golden-chromium/white.jpg deleted file mode 100644 index fb9070def3..0000000000 Binary files a/test/golden-chromium/white.jpg and /dev/null differ diff --git a/test/golden-firefox/grid-cell-0.png b/test/golden-firefox/grid-cell-0.png deleted file mode 100644 index 4677bdbc4f..0000000000 Binary files a/test/golden-firefox/grid-cell-0.png and /dev/null differ diff --git a/test/golden-firefox/grid-cell-1.png b/test/golden-firefox/grid-cell-1.png deleted file mode 100644 index 532dc8db65..0000000000 Binary files a/test/golden-firefox/grid-cell-1.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-clip-odd-size.png b/test/golden-firefox/screenshot-clip-odd-size.png deleted file mode 100644 index b010d1f87f..0000000000 Binary files a/test/golden-firefox/screenshot-clip-odd-size.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-clip-rect.png b/test/golden-firefox/screenshot-clip-rect.png deleted file mode 100644 index ac23b7de50..0000000000 Binary files a/test/golden-firefox/screenshot-clip-rect.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-element-bounding-box.png b/test/golden-firefox/screenshot-element-bounding-box.png deleted file mode 100644 index 32e05bf05b..0000000000 Binary files a/test/golden-firefox/screenshot-element-bounding-box.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-element-fractional.png b/test/golden-firefox/screenshot-element-fractional.png deleted file mode 100644 index 35c53377f9..0000000000 Binary files a/test/golden-firefox/screenshot-element-fractional.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-element-larger-than-viewport.png b/test/golden-firefox/screenshot-element-larger-than-viewport.png deleted file mode 100644 index 5fcdb92355..0000000000 Binary files a/test/golden-firefox/screenshot-element-larger-than-viewport.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-element-padding-border.png b/test/golden-firefox/screenshot-element-padding-border.png deleted file mode 100644 index 917dd48188..0000000000 Binary files a/test/golden-firefox/screenshot-element-padding-border.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-element-scrolled-into-view.png b/test/golden-firefox/screenshot-element-scrolled-into-view.png deleted file mode 100644 index 917dd48188..0000000000 Binary files a/test/golden-firefox/screenshot-element-scrolled-into-view.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-grid-fullpage.png b/test/golden-firefox/screenshot-grid-fullpage.png deleted file mode 100644 index d6d38217f7..0000000000 Binary files a/test/golden-firefox/screenshot-grid-fullpage.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-mobile.png b/test/golden-firefox/screenshot-mobile.png deleted file mode 100644 index 4ba38135af..0000000000 Binary files a/test/golden-firefox/screenshot-mobile.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-offscreen-clip.png b/test/golden-firefox/screenshot-offscreen-clip.png deleted file mode 100644 index 21f7b0f461..0000000000 Binary files a/test/golden-firefox/screenshot-offscreen-clip.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-sanity.png b/test/golden-firefox/screenshot-sanity.png deleted file mode 100644 index ecab61fe17..0000000000 Binary files a/test/golden-firefox/screenshot-sanity.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-translateZ.png b/test/golden-firefox/screenshot-translateZ.png deleted file mode 100644 index e3f5642e60..0000000000 Binary files a/test/golden-firefox/screenshot-translateZ.png and /dev/null differ diff --git a/test/golden-firefox/screenshot-webgl.png b/test/golden-firefox/screenshot-webgl.png deleted file mode 100644 index 76109512b5..0000000000 Binary files a/test/golden-firefox/screenshot-webgl.png and /dev/null differ diff --git a/test/golden-firefox/white.jpg b/test/golden-firefox/white.jpg deleted file mode 100644 index fb9070def3..0000000000 Binary files a/test/golden-firefox/white.jpg and /dev/null differ diff --git a/test/golden-webkit/grid-cell-0.png b/test/golden-webkit/grid-cell-0.png deleted file mode 100644 index 5ae546557b..0000000000 Binary files a/test/golden-webkit/grid-cell-0.png and /dev/null differ diff --git a/test/golden-webkit/grid-cell-1.png b/test/golden-webkit/grid-cell-1.png deleted file mode 100644 index 1110528326..0000000000 Binary files a/test/golden-webkit/grid-cell-1.png and /dev/null differ diff --git a/test/golden-webkit/mock-binary-response.png b/test/golden-webkit/mock-binary-response.png deleted file mode 100644 index 8595e0598e..0000000000 Binary files a/test/golden-webkit/mock-binary-response.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-canvas.png b/test/golden-webkit/screenshot-canvas.png deleted file mode 100644 index dfafbda638..0000000000 Binary files a/test/golden-webkit/screenshot-canvas.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-clip-odd-size.png b/test/golden-webkit/screenshot-clip-odd-size.png deleted file mode 100644 index b010d1f87f..0000000000 Binary files a/test/golden-webkit/screenshot-clip-odd-size.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-clip-rect.png b/test/golden-webkit/screenshot-clip-rect.png deleted file mode 100644 index ac23b7de50..0000000000 Binary files a/test/golden-webkit/screenshot-clip-rect.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-device-scale-factor.png b/test/golden-webkit/screenshot-device-scale-factor.png deleted file mode 100644 index e02fdc8946..0000000000 Binary files a/test/golden-webkit/screenshot-device-scale-factor.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-element-bounding-box.png b/test/golden-webkit/screenshot-element-bounding-box.png deleted file mode 100644 index 32e05bf05b..0000000000 Binary files a/test/golden-webkit/screenshot-element-bounding-box.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-element-fractional-offset.png b/test/golden-webkit/screenshot-element-fractional-offset.png deleted file mode 100644 index 6b65c6fd6b..0000000000 Binary files a/test/golden-webkit/screenshot-element-fractional-offset.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-element-fractional.png b/test/golden-webkit/screenshot-element-fractional.png deleted file mode 100644 index 35c53377f9..0000000000 Binary files a/test/golden-webkit/screenshot-element-fractional.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-element-mobile.png b/test/golden-webkit/screenshot-element-mobile.png deleted file mode 100644 index c2c3ddca29..0000000000 Binary files a/test/golden-webkit/screenshot-element-mobile.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-element-padding-border.png b/test/golden-webkit/screenshot-element-padding-border.png deleted file mode 100644 index 917dd48188..0000000000 Binary files a/test/golden-webkit/screenshot-element-padding-border.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-element-rotate.png b/test/golden-webkit/screenshot-element-rotate.png deleted file mode 100644 index b6eadf8907..0000000000 Binary files a/test/golden-webkit/screenshot-element-rotate.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-element-scrolled-into-view.png b/test/golden-webkit/screenshot-element-scrolled-into-view.png deleted file mode 100644 index 917dd48188..0000000000 Binary files a/test/golden-webkit/screenshot-element-scrolled-into-view.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-grid-fullpage.png b/test/golden-webkit/screenshot-grid-fullpage.png deleted file mode 100644 index d6d38217f7..0000000000 Binary files a/test/golden-webkit/screenshot-grid-fullpage.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-mobile-fullpage.png b/test/golden-webkit/screenshot-mobile-fullpage.png deleted file mode 100644 index bb975e20ef..0000000000 Binary files a/test/golden-webkit/screenshot-mobile-fullpage.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-mobile.png b/test/golden-webkit/screenshot-mobile.png deleted file mode 100644 index 4ba38135af..0000000000 Binary files a/test/golden-webkit/screenshot-mobile.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-offscreen-clip.png b/test/golden-webkit/screenshot-offscreen-clip.png deleted file mode 100644 index 21f7b0f461..0000000000 Binary files a/test/golden-webkit/screenshot-offscreen-clip.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-sanity.png b/test/golden-webkit/screenshot-sanity.png deleted file mode 100644 index ecab61fe17..0000000000 Binary files a/test/golden-webkit/screenshot-sanity.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-translateZ.png b/test/golden-webkit/screenshot-translateZ.png deleted file mode 100644 index e3f5642e60..0000000000 Binary files a/test/golden-webkit/screenshot-translateZ.png and /dev/null differ diff --git a/test/golden-webkit/screenshot-webgl.png b/test/golden-webkit/screenshot-webgl.png deleted file mode 100644 index 76109512b5..0000000000 Binary files a/test/golden-webkit/screenshot-webgl.png and /dev/null differ diff --git a/test/golden-webkit/transparent.png b/test/golden-webkit/transparent.png deleted file mode 100644 index 6c1fe85b52..0000000000 Binary files a/test/golden-webkit/transparent.png and /dev/null differ diff --git a/test/golden-webkit/white.jpg b/test/golden-webkit/white.jpg deleted file mode 100644 index fb9070def3..0000000000 Binary files a/test/golden-webkit/white.jpg and /dev/null differ diff --git a/test/golden-webkit/white.png b/test/golden-webkit/white.png deleted file mode 100644 index e2c01a35a7..0000000000 Binary files a/test/golden-webkit/white.png and /dev/null differ diff --git a/test/headful.spec.ts b/test/headful.spec.ts index 8b860af45d..169fbe0feb 100644 --- a/test/headful.spec.ts +++ b/test/headful.spec.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import './runner/builtin.fixtures'; import './base.fixture'; import utils from './utils'; diff --git a/test/page-screenshot.spec.ts b/test/page-screenshot.spec.ts index 750c148be4..9660d57cc4 100644 --- a/test/page-screenshot.spec.ts +++ b/test/page-screenshot.spec.ts @@ -202,7 +202,7 @@ it.skip(ffheadful)('should work for canvas', async({page, server, golden}) => { await page.setViewportSize({width: 500, height: 500}); await page.goto(server.PREFIX + '/screenshots/canvas.html'); const screenshot = await page.screenshot(); - expect(screenshot).toMatchImage(golden('screenshot-canvas.png')); + expect(screenshot).toMatchImage(golden('screenshot-canvas.png'), { threshold: 0.3 }); }); it.skip(ffheadful)('should work for translateZ', async({page, server, golden}) => { diff --git a/test/pdf.spec.ts b/test/pdf.spec.ts index 47dfca136c..c3d1f4cf8a 100644 --- a/test/pdf.spec.ts +++ b/test/pdf.spec.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import './runner/builtin.fixtures'; import './base.fixture'; import fs from 'fs' diff --git a/test/proxy.spec.ts b/test/proxy.spec.ts index e4999be8ff..65f393c1ff 100644 --- a/test/proxy.spec.ts +++ b/test/proxy.spec.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import './runner/builtin.fixtures'; import './base.fixture'; import socks from 'socksv5'; diff --git a/test/runner/GoldenUtils.js b/test/runner/GoldenUtils.js index ff73e2bc4c..837150aa8b 100644 --- a/test/runner/GoldenUtils.js +++ b/test/runner/GoldenUtils.js @@ -44,7 +44,7 @@ const GoldenComparators = { * @param {!string} mimeType * @return {?{diff: (!Object:undefined), errorMessage: (string|undefined)}} */ -function compareImages(actualBuffer, expectedBuffer, mimeType) { +function compareImages(actualBuffer, expectedBuffer, mimeType, config = {}) { if (!actualBuffer || !(actualBuffer instanceof Buffer)) return { errorMessage: 'Actual result should be Buffer.' }; @@ -56,7 +56,7 @@ function compareImages(actualBuffer, expectedBuffer, mimeType) { }; } const diff = new PNG({width: expected.width, height: expected.height}); - const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, {threshold: 0.2}); + const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, {threshold: 0.2, ...config}); return count > 0 ? { diff: PNG.sync.write(diff) } : null; } @@ -88,8 +88,16 @@ function compareText(actual, expectedBuffer) { * @param {string} path * @return {!{pass: boolean, message: (undefined|string)}} */ -function compare(actual, expectedPath) { +function compare(actual, name, options) { + const { relativeTestFile, snapshotDir, outputDir, updateSnapshots } = options; + let expectedPath; + const testAssetsDir = relativeTestFile.replace(/\.spec\.[jt]s/, ''); + if (path.isAbsolute(name)) + expectedPath = name; + else + expectedPath = path.join(snapshotDir, testAssetsDir, name); if (!fs.existsSync(expectedPath)) { + fs.mkdirSync(path.dirname(expectedPath), { recursive: true }); fs.writeFileSync(expectedPath, actual); return { pass: false, @@ -107,15 +115,35 @@ function compare(actual, expectedPath) { }; } - const result = comparator(actual, expected, mimeType); + const result = comparator(actual, expected, mimeType, options.config); if (!result) return { pass: true }; - const actualPath = addSuffix(expectedPath, '-actual'); - const diffPath = addSuffix(expectedPath, '-diff', result.diffExtension); + if (updateSnapshots) { + fs.mkdirSync(path.dirname(expectedPath), { recursive: true }); + fs.writeFileSync(expectedPath, actual); + return { + pass: true, + message: expectedPath + ' running with --update-snapshots, writing actual.' + }; + } + + let actualPath; + let diffPath; + if (path.isAbsolute(name)) { + actualPath = addSuffix(expectedPath, '-actual'); + diffPath = addSuffix(expectedPath, '-diff', result.diffExtension); + } else { + const outputPath = path.join(outputDir, testAssetsDir, name); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + const expectedPathOut = addSuffix(outputPath, '-expected'); + actualPath = addSuffix(outputPath, '-actual'); + diffPath = addSuffix(outputPath, '-diff', result.diffExtension); + fs.writeFileSync(expectedPathOut, expected); + } fs.writeFileSync(actualPath, actual); if (result.diff) - fs.writeFileSync(diffPath, result.diff); + fs.writeFileSync(diffPath, result.diff); const output = [ c.red(`Image comparison failed:`), diff --git a/test/runner/builtin.fixtures.js b/test/runner/builtin.fixtures.js new file mode 100644 index 0000000000..93d5d13415 --- /dev/null +++ b/test/runner/builtin.fixtures.js @@ -0,0 +1,48 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const util = require('util'); +const { mkdtempAsync, removeFolderAsync } = require('../utils'); +const { registerFixture, registerWorkerFixture } = require('./fixtures'); +const mkdirAsync = util.promisify(fs.mkdir.bind(fs)); + +let workerId; +let outputDir; + +registerWorkerFixture('parallelIndex', async ({}, test) => { + await test(workerId); +}); + +registerFixture('tmpDir', async ({}, test) => { + const tmpDir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-')); + await test(tmpDir); + await removeFolderAsync(tmpDir).catch(e => {}); +}); + +registerWorkerFixture('outputDir', async ({}, test) => { + await mkdirAsync(outputDir, { recursive: true }); + await test(outputDir); +}); + +function initializeWorker(options) { + workerId = options.workerId; + outputDir = options.outputDir; +} + +module.exports = { initializeWorker }; diff --git a/test/runner/checkCoverage.js b/test/runner/checkCoverage.js index 6cda377299..8db519b333 100644 --- a/test/runner/checkCoverage.js +++ b/test/runner/checkCoverage.js @@ -38,7 +38,7 @@ if (browserName !== 'chromium') { if (browserName === 'webkit') api.delete('browserContext.clearPermissions'); -const coverageDir = path.join(__dirname, '..', 'output-' + browserName, 'coverage'); +const coverageDir = path.join(__dirname, '..', 'coverage-report', 'coverage'); const coveredMethods = new Set(); for (const file of getCoverageFiles(coverageDir)) { diff --git a/test/runner/fixturesUI.js b/test/runner/fixturesUI.js index 643567449c..235e1b7027 100644 --- a/test/runner/fixturesUI.js +++ b/test/runner/fixturesUI.js @@ -23,7 +23,6 @@ Error.stackTraceLimit = 15; global.testOptions = require('./testOptions'); global.registerFixture = registerFixture; global.registerWorkerFixture = registerWorkerFixture; -process.env.JEST_WORKER_ID = 1; const fixturePool = new FixturePool(); let revertBabelRequire; diff --git a/test/runner/index.js b/test/runner/index.js index 02e68c9e42..e09ee6ab87 100644 --- a/test/runner/index.js +++ b/test/runner/index.js @@ -31,10 +31,13 @@ program .option('--trial-run', 'Only collect the matching tests and report them as passing') .option('--quiet', 'Suppress stdio', false) .option('--debug', 'Run tests in-process for debugging', false) + .option('--output ', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results')) .option('--timeout ', 'Specify test timeout threshold (in milliseconds), default: 10000', 10000) + .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run') .action(async (command) => { - // Collect files - const files = collectFiles(path.join(process.cwd(), command.args[0]), command.args.slice(1)); + // Collect files] + const testDir = path.join(process.cwd(), command.args[0]); + const files = collectFiles(testDir, '', command.args.slice(1)); const rootSuite = new createTestSuite(); let total = 0; @@ -44,6 +47,7 @@ program forbidOnly: command.forbidOnly || undefined, grep: command.grep, reporter: NullReporter, + testDir, timeout: command.timeout, trialRun: true, }); @@ -74,10 +78,14 @@ program quiet: command.quiet, grep: command.grep, jobs, + outputDir: command.output, reporter: command.reporter, retries: command.retries, + snapshotDir: path.join(testDir, '__snapshots__'), + testDir, timeout: command.timeout, trialRun: command.trialRun, + updateSnapshots: command.updateSnapshots }); await runner.run(files); await runner.stop(); @@ -86,24 +94,26 @@ program program.parse(process.argv); -function collectFiles(dir, filters) { - if (fs.statSync(dir).isFile()) - return [dir]; +function collectFiles(testDir, dir, filters) { + const fullDir = path.join(testDir, dir); + if (fs.statSync(fullDir).isFile()) + return [fullDir]; const files = []; - for (const name of fs.readdirSync(dir)) { - if (fs.lstatSync(path.join(dir, name)).isDirectory()) { - files.push(...collectFiles(path.join(dir, name), filters)); + for (const name of fs.readdirSync(fullDir)) { + if (fs.lstatSync(path.join(fullDir, name)).isDirectory()) { + files.push(...collectFiles(testDir, path.join(dir, name), filters)); continue; } if (!name.endsWith('spec.ts')) continue; + const relativeName = path.join(dir, name); + const fullName = path.join(testDir, relativeName); if (!filters.length) { - files.push(path.join(dir, name)); + files.push(fullName); continue; } - const fullName = path.join(dir, name); for (const filter of filters) { - if (fullName.includes(filter)) { + if (relativeName.includes(filter)) { files.push(fullName); break; } diff --git a/test/runner/runner.js b/test/runner/runner.js index e0741f56ba..ea9b9ac741 100644 --- a/test/runner/runner.js +++ b/test/runner/runner.js @@ -20,7 +20,7 @@ const { EventEmitter } = require('events'); const Mocha = require('mocha'); const builtinReporters = require('mocha/lib/reporters'); const DotRunner = require('./dotReporter'); -const { computeWorkerHash, FixturePool } = require('./fixtures'); +const { computeWorkerHash } = require('./fixtures'); const constants = Mocha.Runner.constants; // Mocha runner does not remove uncaughtException listeners. @@ -225,7 +225,7 @@ class OopWorker extends EventEmitter { } async init() { - this.process.send({ method: 'init', params: { workerId: lastWorkerId++ } }); + this.process.send({ method: 'init', params: { workerId: lastWorkerId++, ...this.runner._options } }); await new Promise(f => this.process.once('message', f)); // Ready ack } @@ -259,6 +259,10 @@ class InProcessWorker extends EventEmitter { } async init() { + const { initializeImageMatcher } = require('./testRunner'); + const { initializeWorker } = require('./builtin.fixtures'); + initializeImageMatcher(this.runner._options); + initializeWorker({ ...this.runner._options.outputDir, workerId: 0 }); } async run(entry) { diff --git a/test/runner/testRunner.js b/test/runner/testRunner.js index ca46de001f..b64aeb1e7b 100644 --- a/test/runner/testRunner.js +++ b/test/runner/testRunner.js @@ -14,6 +14,7 @@ * limitations under the License. */ +const path = require('path'); const Mocha = require('mocha'); const { fixturesUI } = require('./fixturesUI'); const { EventEmitter } = require('events'); @@ -22,14 +23,6 @@ global.expect = require('expect'); global.testOptions = require('./testOptions'); const GoldenUtils = require('./GoldenUtils'); -(function extendExpects() { - function toMatchImage(received, path) { - const {pass, message} = GoldenUtils.compare(received, path); - return {pass, message: () => message}; - }; - global.expect.extend({ toMatchImage }); -})(); - class NullReporter {} class TestRunner extends EventEmitter { @@ -50,7 +43,7 @@ class TestRunner extends EventEmitter { this._passes = 0; this._failures = 0; this._pending = 0; - + this._relativeTestFile = path.relative(options.testDir, file); this.mocha.addFile(file); this.mocha.suite.filterOnly(); this.mocha.loadFiles(); @@ -65,6 +58,7 @@ class TestRunner extends EventEmitter { const constants = Mocha.Runner.constants; runner.on(constants.EVENT_TEST_BEGIN, test => { + relativeTestFile = this._relativeTestFile; if (this._failedWithError) { ++remaining; return; @@ -185,4 +179,14 @@ function serializeError(error) { return trimCycles(error); } -module.exports = { TestRunner, createTestSuite }; +let relativeTestFile; + +function initializeImageMatcher(options) { + function toMatchImage(received, name, config) { + const { pass, message } = GoldenUtils.compare(received, name, { ...options, relativeTestFile, config }); + return { pass, message: () => message }; + }; + global.expect.extend({ toMatchImage }); +} + +module.exports = { TestRunner, createTestSuite, initializeImageMatcher }; diff --git a/test/runner/worker.js b/test/runner/worker.js index 41eb00cb3a..001ce4cfa9 100644 --- a/test/runner/worker.js +++ b/test/runner/worker.js @@ -17,7 +17,9 @@ const debug = require('debug'); const { fixturePool } = require('./fixturesUI'); const { gracefullyCloseAll } = require('../../lib/server/processLauncher'); -const { TestRunner } = require('./testRunner'); +const { TestRunner, initializeImageMatcher } = require('./testRunner'); +const { initializeWorker } = require('./builtin.fixtures'); + const util = require('util'); let closed = false; @@ -49,12 +51,17 @@ process.on('SIGINT',() => {}); process.on('SIGTERM',() => {}); process.on('message', async message => { - if (message.method === 'init') - process.env.JEST_WORKER_ID = message.params.workerId; + if (message.method === 'init') { + initializeWorker(message.params); + initializeImageMatcher(message.params); + return; + } if (message.method === 'stop') { await fixturePool.teardownScope('worker'); await gracefullyCloseAndExit(); - } if (message.method === 'run') { + return; + } + if (message.method === 'run') { const testRunner = new TestRunner(message.params.file, message.params.startOrdinal, message.params.options); for (const event of ['test', 'pending', 'pass', 'fail', 'done']) testRunner.on(event, sendMessageToParent.bind(null, event)); diff --git a/test/screencast.spec.ts b/test/screencast.spec.ts index 6bffb979cb..330bb3add5 100644 --- a/test/screencast.spec.ts +++ b/test/screencast.spec.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import './runner/builtin.fixtures'; import './base.fixture'; import { registerFixture } from './runner/fixtures'; import { Page } from '..'; diff --git a/test/types.d.ts b/test/types.d.ts index 0398334ba2..d51fcec8e3 100644 --- a/test/types.d.ts +++ b/test/types.d.ts @@ -1,3 +1,19 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + type ServerResponse = import('http').ServerResponse; type IncomingMessage = import('http').IncomingMessage; @@ -11,6 +27,8 @@ type ItFunction = ((name: string, inner: (state: STATE) => Promise) }; interface WorkerState { + parallelIndex: number; + tmpDir: string; } interface FixtureState { @@ -19,7 +37,7 @@ interface FixtureState { declare module '' { module 'expect/build/types' { interface Matchers { - toMatchImage(path: string): R; + toMatchImage(path: string, options?: { threshold?: number }): R; } } }