test: add component tests for html report (#10900)

.github/workflows/tests_primary.yml

@@ -76,3 +76,21 @@ jobs:
     - run: xvfb-run npm run ttest
       if: matrix.os == 'ubuntu-latest'
 
+  test_html_report:
+    name: HTML Report
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: 12
+    - run: npm i -g npm@8
+    - run: npm ci
+      env:
+        DEBUG: pw:install
+    - run: npm run build
+    - run: npx playwright install --with-deps
+    - run: npm run htest
+      if: matrix.os != 'ubuntu-latest'
+    - run: xvfb-run npm run htest
+      if: matrix.os == 'ubuntu-latest'

package.json

@@ -18,6 +18,7 @@
     "wtest": "playwright test --config=tests/config/default.config.ts --project=webkit",
     "atest": "playwright test --config=tests/config/android.config.ts",
     "etest": "playwright test --config=tests/config/electron.config.ts",
+    "htest": "playwright test --config=packages/html-reporter",
     "ttest": "node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright-test.config.ts",
     "vtest": "cross-env PLAYWRIGHT_DOCKER=1 node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright-test.config.ts",
     "test": "playwright test --config=tests/config/default.config.ts",

packages/html-reporter/playwright.config.ts

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { PlaywrightTestConfig, devices } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  testDir: 'src',
+  snapshotDir: 'snapshots',
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  use: {
+    trace: 'on-first-retry',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+};
+
+export default config;

packages/html-reporter/playwright.stories.tsx

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './src/theme.css';
+import './src/chip.story.tsx';
+import './src/headerView.story.tsx';

packages/html-reporter/src/chip.spec.ts

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from '../test/componentTest';
+
+test.use({ webpack: require.resolve('../webpack.config.js') });
+
+test('chip expand collapse', async ({ renderComponent }) => {
+  const component = await renderComponent('ChipComponent');
+  await expect(component.locator('text=Chip body')).toBeVisible();
+  // expect(await component.screenshot()).toMatchSnapshot('expanded.png');
+  await component.locator('text=Title').click();
+  await expect(component.locator('text=Chip body')).not.toBeVisible();
+  // expect(await component.screenshot()).toMatchSnapshot('collapsed.png');
+  await component.locator('text=Title').click();
+  await expect(component.locator('text=Chip body')).toBeVisible();
+  // expect(await component.screenshot()).toMatchSnapshot('expanded.png');
+});
+
+test('chip render long title', async ({ renderComponent }) => {
+  const title = 'Extremely long title. '.repeat(10);
+  const component = await renderComponent('ChipComponent', { title });
+  await expect(component).toContainText('Extremely long title.');
+  await expect(component.locator('text=Extremely long title.')).toHaveAttribute('title', title);
+});
+
+test('chip setExpanded is called', async ({ renderComponent }) => {
+  const expandedValues: boolean[] = [];
+  const component = await renderComponent('ChipComponentWithFunctions', {
+    setExpanded: (expanded: boolean) => expandedValues.push(expanded)
+  });
+
+  await component.locator('text=Title').click();
+  expect(expandedValues).toEqual([true]);
+});

packages/html-reporter/src/chip.story.tsx

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { Chip } from './chip';
+import { registerComponent } from '../test/component';
+
+const ChipComponent: React.FC<{
+  title?: string
+}> = ({ title }) => {
+  const [expanded, setExpanded] = React.useState(true);
+  return <Chip header={title || 'Title'} expanded={expanded} setExpanded={setExpanded}>
+    Chip body
+  </Chip>;
+};
+registerComponent('ChipComponent', ChipComponent, {
+  viewport: { width: 500, height: 500 },
+});
+
+const ChipComponentWithFunctions: React.FC<{
+  setExpanded: (expanded: boolean) => void,
+}> = ({ setExpanded }) => {
+  return <Chip header='Title' expanded={false} setExpanded={setExpanded}>
+    Chip body
+  </Chip>;
+};
+registerComponent('ChipComponentWithFunctions', ChipComponentWithFunctions, {
+  viewport: { width: 500, height: 500 },
+});

packages/html-reporter/src/chip.tsx

@@ -16,6 +16,8 @@
 
 import * as React from 'react';
 import './chip.css';
+import './colors.css';
+import './common.css';
 import * as icons from './icons';
 
 export const Chip: React.FunctionComponent<{
@@ -26,7 +28,10 @@ export const Chip: React.FunctionComponent<{
   children?: any,
 }> = ({ header, expanded, setExpanded, children, noInsets }) => {
   return <div className='chip'>
-    <div className={'chip-header' + (setExpanded ? ' expanded-' + expanded : '')} onClick={() => setExpanded?.(!expanded)}>
+    <div
+      className={'chip-header' + (setExpanded ? ' expanded-' + expanded : '')}
+      onClick={() => setExpanded?.(!expanded)}
+      title={typeof header === 'string' ? header : undefined}>
       {setExpanded && !!expanded && icons.downArrow()}
       {setExpanded && !expanded && icons.rightArrow()}
       {header}

packages/html-reporter/src/common.css

@@ -14,6 +14,16 @@
   limitations under the License.
 */
 
+:root {
+  --box-shadow: rgba(0, 0, 0, 0.133) 0px 1.6px 3.6px 0px, rgba(0, 0, 0, 0.11) 0px 0.3px 0.9px 0px;
+  --monospace-font: "SF Mono", Monaco, Consolas, "Droid Sans Mono", Inconsolata, "Courier New",monospace;
+  --box-shadow-thick: rgb(0 0 0 / 10%) 0px 1.8px 1.9px,
+    rgb(0 0 0 / 15%) 0px 6.1px 6.3px,
+    rgb(0 0 0 / 10%) 0px -2px 4px,
+    rgb(0 0 0 / 15%) 0px -6.1px 12px,
+    rgb(0 0 0 / 25%) 0px 6px 12px;
+}
+
 * {
   box-sizing: border-box;
   min-width: 0;

packages/html-reporter/src/headerView.css

@@ -0,0 +1,32 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+.header-view-status-container {
+  float: right;
+}
+
+@media only screen and (max-width: 600px) {
+  .header-view-status-container {
+    float: none;
+    margin: 0 0 10px 0 !important;
+    overflow: hidden;
+  }
+
+  .header-view-status-container .subnav-search-input {
+    border-left: none;
+    border-right: none;
+  }
+}

packages/html-reporter/src/headerView.spec.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Stats } from '@playwright/test/src/reporters/html';
+import { test, expect } from '../test/componentTest';
+
+test.use({ webpack: require.resolve('../webpack.config.js') });
+
+test('should render counters', async ({ renderComponent }) => {
+  const stats: Stats = {
+    total: 100,
+    expected: 42,
+    unexpected: 31,
+    flaky: 17,
+    skipped: 10,
+    ok: false,
+    duration: 100000
+  };
+  const component = await renderComponent('HeaderView', { stats });
+  await expect(component.locator('a').withText('All').locator('.counter')).toHaveText('100');
+  await expect(component.locator('a').withText('Passed').locator('.counter')).toHaveText('42');
+  await expect(component.locator('a').withText('Failed').locator('.counter')).toHaveText('31');
+  await expect(component.locator('a').withText('Flaky').locator('.counter')).toHaveText('17');
+  await expect(component.locator('a').withText('Skipped').locator('.counter')).toHaveText('10');
+});
+
+test('should toggle filters', async ({ page, renderComponent }) => {
+  const stats: Stats = {
+    total: 100,
+    expected: 42,
+    unexpected: 31,
+    flaky: 17,
+    skipped: 10,
+    ok: false,
+    duration: 100000
+  };
+  const filters: string[] = [];
+  const component = await renderComponent('HeaderView', {
+    stats,
+    setFilterText: (filterText: string) => filters.push(filterText)
+  });
+  await component.locator('a').withText('All').click();
+  await component.locator('a').withText('Passed').click();
+  await expect(page).toHaveURL(/#\?q=s:passed/);
+  await component.locator('a').withText('Failed').click();
+  await expect(page).toHaveURL(/#\?q=s:failed/);
+  await component.locator('a').withText('Flaky').click();
+  await expect(page).toHaveURL(/#\?q=s:flaky/);
+  await component.locator('a').withText('Skipped').click();
+  await expect(page).toHaveURL(/#\?q=s:skipped/);
+  expect(filters).toEqual(['', 's:passed', 's:failed', 's:flaky', 's:skipped']);
+});

packages/html-reporter/src/headerView.story.tsx

@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { registerComponent } from '../test/component';
+import { HeaderView } from './headerView';
+
+registerComponent('HeaderView', HeaderView, {
+  viewport: { width: 720, height: 500 },
+});

packages/html-reporter/src/headerView.tsx

@@ -16,7 +16,9 @@
 
 import type { Stats } from '@playwright/test/src/reporters/html';
 import * as React from 'react';
+import './colors.css';
 import './common.css';
+import './headerView.css';
 import * as icons from './icons';
 import { Link, navigate } from './links';
 import { statusIcon } from './statusIcon';
@@ -36,7 +38,7 @@ export const HeaderView: React.FC<{
   });
 
   return <div className='pt-3'>
-    <div className='status-container ml-2 pl-2 d-flex'>
+    <div className='header-view-status-container ml-2 pl-2 d-flex'>
       <StatsNavView stats={stats}></StatsNavView>
     </div>
     <form className='subnav-search' onSubmit={

packages/html-reporter/src/reportView.css

@@ -14,25 +14,6 @@
   limitations under the License.
 */
 
-:root {
-  --box-shadow: rgba(0, 0, 0, 0.133) 0px 1.6px 3.6px 0px, rgba(0, 0, 0, 0.11) 0px 0.3px 0.9px 0px;
-  --monospace-font: "SF Mono", Monaco, Consolas, "Droid Sans Mono", Inconsolata, "Courier New",monospace;
-  --box-shadow-thick: rgb(0 0 0 / 10%) 0px 1.8px 1.9px,
-    rgb(0 0 0 / 15%) 0px 6.1px 6.3px,
-    rgb(0 0 0 / 10%) 0px -2px 4px,
-    rgb(0 0 0 / 15%) 0px -6.1px 12px,
-    rgb(0 0 0 / 25%) 0px 6px 12px;
-}
-
-#root {
-  width: 100%;
-  height: 100%;
-  color: var(--color-fg-default);
-  font-size: 14px;
-  font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";
-  -webkit-font-smoothing: antialiased;
-}
-
 html, body {
   width: 100%;
   height: 100%;
@@ -52,23 +33,8 @@ body {
   border-top: 1px solid var(--color-border-default);
 }
 
-.status-container {
-  float: right;
-}
-
 @media only screen and (max-width: 600px) {
   .report {
     padding: 0 !important;
   }
-
-  .status-container {
-    float: none;
-    margin: 0 0 10px 0 !important;
-    overflow: hidden;
-  }
-
-  .subnav-search-input {
-    border-left: none;
-    border-right: none;
-  }
 }

packages/html-reporter/src/reportView.tsx

@@ -25,6 +25,7 @@ import { LoadedReport } from './loadedReport';
 import './reportView.css';
 import { TestCaseView } from './testCaseView';
 import { TestFilesView } from './testFilesView';
+import './theme.css';
 
 declare global {
   interface Window {

packages/html-reporter/src/theme.css

@@ -0,0 +1,22 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+#root {
+  color: var(--color-fg-default);
+  font-size: 14px;
+  font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";
+  -webkit-font-smoothing: antialiased;
+}

packages/html-reporter/test/component.js

@@ -0,0 +1,77 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const React = require('react');
+const ReactDOM = require('react-dom');
+
+const fillStyle = {
+  position: 'absolute',
+  top: 0,
+  right: 0,
+  bottom: 0,
+  left: 0,
+};
+
+const checkerboardCommon = {
+  ...fillStyle,
+  backgroundSize: '50px 50px',
+  backgroundPosition: '0 0, 25px 25px',
+};
+
+const checkerboardLight = {
+  ...checkerboardCommon,
+  backgroundColor: '#FFF',
+  backgroundImage: `linear-gradient(45deg, #00000008 25%, transparent 25%, transparent 75%, #00000008 75%, #00000008),
+                    linear-gradient(45deg, #00000008 25%, transparent 25%, transparent 75%, #00000008 75%, #00000008)`
+};
+
+const checkerboardDark = {
+  ...checkerboardCommon,
+  backgroundColor: '#000',
+  backgroundImage: `linear-gradient(45deg, #FFFFFF12 25%, transparent 25%, transparent 75%, #FFFFFF12 75%, #FFFFFF12),
+                    linear-gradient(45deg, #FFFFFF12 25%, transparent 25%, transparent 75%, #FFFFFF12 75%, #FFFFFF12)`
+};
+
+const Component = ({ style, children }) => {
+  const checkerboard = window.matchMedia('(prefers-color-scheme: dark)').matches ? checkerboardDark : checkerboardLight;
+  const bgStyle = { ...checkerboard };
+  const fgStyle = { ...fillStyle, ...style };
+  return React.createElement(
+      React.Fragment, null,
+      React.createElement('div', { style: bgStyle }),
+      React.createElement('div', { style: fgStyle, id: 'pw-root' }, children));
+};
+
+const registry = new Map();
+
+export const registerComponent = (name, component, options) => {
+  registry.set(name, { component, options });
+};
+
+function render(name, params) {
+  const entry = registry.get(name);
+  ReactDOM.render(
+      React.createElement(Component, null, React.createElement(entry.component, params || null)),
+      document.getElementById('root'));
+}
+
+function options(name) {
+  const entry = registry.get(name);
+  return entry.options;
+}
+
+window.__playwright_render = render;
+window.__playwright_options = options;

packages/html-reporter/test/componentTest.ts

@@ -0,0 +1,78 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as path from 'path';
+import { test as baseTest, Locator } from '@playwright/test';
+
+declare global {
+  interface Window {
+    __playwright_render: (component: string, props: any) => void;
+    __playwright_options: (component: string) => { viewport: { width: number, height: number } };
+  }
+}
+
+type TestFixtures = {
+  renderComponent: (component: string, params?: any) => Promise<Locator>;
+  webpack: string;
+};
+
+export const test = baseTest.extend<TestFixtures>({
+  webpack: '',
+  renderComponent: async ({ page, webpack }, use, testInfo) => {
+    const webpackConfig = require(webpack);
+    const outputPath = webpackConfig.output.path;
+    const filename = webpackConfig.output.filename.replace('[name]', 'playwright');
+    await use(async (component: string, optionalParams?: Object) => {
+      await page.route('http://component/index.html', route => {
+        route.fulfill({
+          body: `<html>
+            <meta name='color-scheme' content='dark light'>
+            <style>html, body { padding: 0; margin: 0; background: #aaa; }</style>
+            <div id='root' style='width: 100%; height: 100%;'></div>
+          </html>`,
+          contentType: 'text/html'
+        });
+      });
+      await page.goto('http://component/index.html');
+
+      await page.addScriptTag({ path: path.resolve(__dirname, outputPath, filename) });
+      const options = await page.evaluate((component: string) => {
+        return window.__playwright_options(component);
+      }, component);
+      await page.setViewportSize(options.viewport);
+
+      const params = { ...optionalParams };
+      for (const [key, value] of Object.entries(params)) {
+        if (typeof value === 'function') {
+          const functionName = '__pw_func_' + key;
+          await page.exposeFunction(functionName, value);
+          (params as any)[key] = functionName;
+        }
+      }
+      await page.evaluate(v => {
+        const params = v.params;
+        for (const [key, value] of Object.entries(params)) {
+          if (typeof value === 'string' && (value as string).startsWith('__pw_func_'))
+            (params as any)[key] = window[value];
+        }
+        window.__playwright_render(v.component, params);
+      }, { component, params });
+      return page.locator('#pw-root');
+    });
+  },
+});
+
+export { expect } from '@playwright/test';

packages/html-reporter/webpack.config.js

@@ -25,6 +25,7 @@ module.exports = {
   entry: {
     zip: require.resolve('@zip.js/zip.js/dist/zip-no-worker-inflate.min.js'),
     app: path.join(__dirname, 'src', 'index.tsx'),
+    playwright: path.join(__dirname, 'playwright.stories.tsx'),
   },
   resolve: {
     extensions: ['.ts', '.js', '.tsx', '.jsx']
@@ -59,6 +60,7 @@ module.exports = {
       title: 'Playwright Test Report',
       template: path.join(__dirname, 'src', 'index.html'),
       inject: true,
+      excludeChunks: ['playwright'],
     }),
     new BundleJsPlugin(),
   ]