feat(test): start authoring raw reporter (#8790)

.gitignore

@@ -19,3 +19,4 @@ nohup.out
 .tmp
 allure*
 playwright-report
+/demo/

src/cli/cli.ts

@@ -237,6 +237,8 @@ if (!process.env.PW_CLI_TARGET_LANG) {
 
   if (playwrightTestPackagePath) {
     require(playwrightTestPackagePath).addTestCommand(program);
+    if (process.env.PW_EXPERIMENTAL)
+      require(playwrightTestPackagePath).addGenerateHtmlCommand(program);
   } else {
     const command = program.command('test').allowUnknownOption(true);
     command.description('Run tests with Playwright Test. Available in @playwright/test package.');

src/test/cli.ts

@@ -23,6 +23,8 @@ import type { Config } from './types';
 import { Runner, builtInReporters, BuiltInReporter } from './runner';
 import { stopProfiling, startProfiling } from './profiler';
 import { FilePatternFilter } from './util';
+import { Loader } from './loader';
+import { HtmlBuilder } from './html/htmlBuilder';
 
 const defaultTimeout = 30000;
 const defaultReporter: BuiltInReporter = process.env.CI ? 'dot' : 'list';
@@ -81,9 +83,31 @@ export function addTestCommand(program: commander.CommanderStatic) {
   });
 }
 
-async function runTests(args: string[], opts: { [key: string]: any }) {
-  await startProfiling();
+export function addGenerateHtmlCommand(program: commander.CommanderStatic) {
+  const command = program.command('generate-html');
+  command.description('Generate HTML report');
+  command.option('-c, --config <file>', `Configuration file, or a test directory with optional "${tsConfig}"/"${jsConfig}"`);
+  command.option('--output <dir>', `Folder for output artifacts (default: "playwright-report")`, 'playwright-report');
+  command.action(async opts => {
+    const loader = await createLoader(opts);
+    const outputFolders = new Set(loader.projects().map(p => p.config.outputDir));
+    const reportFiles = new Set<string>();
+    for (const outputFolder of outputFolders) {
+      const reportFolder = path.join(outputFolder, 'report');
+      const files = fs.readdirSync(reportFolder).filter(f => f.endsWith('.report'));
+      for (const file of files)
+        reportFiles.add(path.join(reportFolder, file));
+    }
+    new HtmlBuilder([...reportFiles], opts.output);
+  }).on('--help', () => {
+    console.log('');
+    console.log('Examples:');
+    console.log('');
+    console.log('  $ generate-report');
+  });
+}
 
+async function createLoader(opts: { [key: string]: any }): Promise<Loader> {
   if (opts.browser) {
     const browserOpt = opts.browser.toLowerCase();
     if (!['all', 'chromium', 'firefox', 'webkit'].includes(browserOpt))
@@ -100,13 +124,13 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
   const overrides = overridesFromOptions(opts);
   if (opts.headed)
     overrides.use = { headless: false };
-  const runner = new Runner(defaultConfig, overrides);
+  const loader = new Loader(defaultConfig, overrides);
 
   async function loadConfig(configFile: string) {
     if (fs.existsSync(configFile)) {
       if (process.stdout.isTTY)
         console.log(`Using config at ` + configFile);
-      const loadedConfig = await runner.loadConfigFile(configFile);
+      const loadedConfig = await loader.loadConfigFile(configFile);
       if (('projects' in loadedConfig) && opts.browser)
         throw new Error(`Cannot use --browser option when configuration file defines projects. Specify browserName in the projects instead.`);
       return true;
@@ -131,7 +155,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
       // When passed a directory, look for a config file inside.
       if (!await loadConfigFromDirectory(configFile)) {
         // If there is no config, assume this as a root testing directory.
-        runner.loadEmptyConfig(configFile);
+        loader.loadEmptyConfig(configFile);
       }
     } else {
       // When passed a file, it must be a config file.
@@ -140,9 +164,15 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
   } else if (!await loadConfigFromDirectory(process.cwd())) {
     // No --config option, let's look for the config file in the current directory.
     // If not, scan the world.
-    runner.loadEmptyConfig(process.cwd());
+    loader.loadEmptyConfig(process.cwd());
   }
+  return loader;
+}
 
+async function runTests(args: string[], opts: { [key: string]: any }) {
+  await startProfiling();
+
+  const loader = await createLoader(opts);
   const filePatternFilters: FilePatternFilter[] = args.map(arg => {
     const match = /^(.*):(\d+)$/.exec(arg);
     return {
@@ -150,6 +180,8 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
       line: match ? parseInt(match[2], 10) : null,
     };
   });
+
+  const runner = new Runner(loader);
   const result = await runner.run(!!opts.list, filePatternFilters, opts.project || undefined);
   await stopProfiling(undefined);
 

src/test/html/htmlBuilder.ts

@@ -0,0 +1,101 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { ProjectTreeItem, SuiteTreeItem, TestTreeItem, TestCase, TestResult, TestStep } from './types';
+import { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../reporters/raw';
+
+export class HtmlBuilder {
+  private _reportFolder: string;
+  private _tests = new Map<string, JsonTestCase>();
+
+  constructor(rawReports: string[], outputDir: string) {
+    this._reportFolder = path.resolve(process.cwd(), outputDir);
+    const dataFolder = path.join(this._reportFolder, 'data');
+    fs.mkdirSync(dataFolder, { recursive: true });
+    const appFolder = path.join(__dirname, '..', '..', 'web', 'htmlReport2');
+    for (const file of fs.readdirSync(appFolder))
+      fs.copyFileSync(path.join(appFolder, file), path.join(this._reportFolder, file));
+    const projects: ProjectTreeItem[] = rawReports.map(rawReport => {
+      const json = JSON.parse(fs.readFileSync(rawReport, 'utf-8')) as JsonReport;
+      const suits = json.suites.map(s => this._createSuiteTreeItem(s));
+      return {
+        name: json.project.name,
+        suits,
+        failedTests: suits.reduce((a, s) => a + s.failedTests, 0)
+      };
+    });
+    fs.writeFileSync(path.join(dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2));
+
+    for (const [testId, test] of this._tests) {
+      const testCase: TestCase = {
+        testId: test.testId,
+        title: test.title,
+        location: test.location,
+        results: test.results.map(r => this._createTestResult(r))
+      };
+      fs.writeFileSync(path.join(dataFolder, testId + '.json'), JSON.stringify(testCase, undefined, 2));
+    }
+  }
+
+  private _createSuiteTreeItem(suite: JsonSuite): SuiteTreeItem {
+    const suites = suite.suites.map(s => this._createSuiteTreeItem(s));
+    const tests = suite.tests.map(t => this._createTestTreeItem(t));
+    return {
+      title: suite.title,
+      location: suite.location,
+      duration: suites.reduce((a, s) => a + s.duration, 0) + tests.reduce((a, t) => a + t.duration, 0),
+      failedTests: suites.reduce((a, s) => a + s.failedTests, 0) + tests.reduce((a, t) => t.outcome === 'unexpected' || t.outcome === 'flaky' ? a + 1 : a, 0),
+      suites,
+      tests
+    };
+  }
+
+  private _createTestTreeItem(test: JsonTestCase): TestTreeItem {
+    const duration = test.results.reduce((a, r) => a + r.duration, 0);
+    this._tests.set(test.testId, test);
+    return {
+      testId: test.testId,
+      location: test.location,
+      title: test.title,
+      duration,
+      outcome: test.outcome
+    };
+  }
+
+  private _createTestResult(result: JsonTestResult): TestResult {
+    return {
+      duration: result.duration,
+      startTime: result.startTime,
+      retry: result.retry,
+      steps: result.steps.map(s => this._createTestStep(s)),
+      error: result.error,
+      status: result.status,
+    };
+  }
+
+  private _createTestStep(step: JsonTestStep): TestStep {
+    return {
+      title: step.title,
+      startTime: step.startTime,
+      duration: step.duration,
+      steps: step.steps.map(s => this._createTestStep(s)),
+      log: step.log,
+      error: step.error
+    };
+  }
+}

src/test/html/types.ts

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type Location = {
+  file: string;
+  line: number;
+  column: number;
+};
+
+export type ProjectTreeItem = {
+  name: string;
+  suits: SuiteTreeItem[];
+  failedTests: number;
+};
+
+export type SuiteTreeItem = {
+  title: string;
+  location?: Location;
+  duration: number;
+  suites: SuiteTreeItem[];
+  tests: TestTreeItem[];
+  failedTests: number;
+};
+
+export type TestTreeItem = {
+  testId: string,
+  title: string;
+  location: Location;
+  duration: number;
+  outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';
+};
+
+export type TestCase = {
+  testId: string,
+  title: string;
+  location: Location;
+  results: TestResult[];
+};
+
+export interface TestError {
+  message?: string;
+  stack?: string;
+  value?: string;
+}
+
+export type TestResult = {
+  retry: number;
+  startTime: string;
+  duration: number;
+  steps: TestStep[];
+  error?: TestError;
+  status: 'passed' | 'failed' | 'timedOut' | 'skipped';
+};
+
+export type TestStep = {
+  title: string;
+  startTime: string;
+  duration: number;
+  log?: string[];
+  error?: TestError;
+  steps: TestStep[];
+};

src/test/reporters/raw.ts

@@ -0,0 +1,244 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { FullProject } from '../../../types/test';
+import { FullConfig, Location, Suite, TestCase, TestError, TestResult, TestStatus, TestStep } from '../../../types/testReporter';
+import { assert, calculateSha1 } from '../../utils/utils';
+import { sanitizeForFilePath } from '../util';
+import { serializePatterns, toPosixPath } from './json';
+
+export type JsonStats = { expected: number, unexpected: number, flaky: number, skipped: number };
+export type JsonLocation = Location;
+export type JsonStackFrame = { file: string, line: number, column: number };
+
+export type JsonReport = {
+  config: JsonConfig,
+  project: JsonProject,
+  suites: JsonSuite[],
+};
+
+export type JsonConfig = Omit<FullConfig, 'projects'>;
+
+export type JsonProject = {
+  metadata: any,
+  name: string,
+  outputDir: string,
+  repeatEach: number,
+  retries: number,
+  testDir: string,
+  testIgnore: string[],
+  testMatch: string[],
+  timeout: number,
+};
+
+export type JsonSuite = {
+  title: string;
+  location?: JsonLocation;
+  suites: JsonSuite[];
+  tests: JsonTestCase[];
+};
+
+export type JsonTestCase = {
+  testId: string;
+  title: string;
+  location: JsonLocation;
+  expectedStatus: TestStatus;
+  timeout: number;
+  annotations: { type: string, description?: string }[];
+  retries: number;
+  results: JsonTestResult[];
+  ok: boolean;
+  outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';
+};
+
+export type TestAttachment = {
+  name: string;
+  path?: string;
+  body?: Buffer;
+  contentType: string;
+};
+
+export type JsonAttachment = {
+  name: string;
+  path: string;
+  contentType: string;
+};
+
+export type JsonTestResult = {
+  retry: number;
+  workerIndex: number;
+  startTime: string;
+  duration: number;
+  status: TestStatus;
+  error?: TestError;
+  attachments: JsonAttachment[];
+  steps: JsonTestStep[];
+};
+
+export type JsonTestStep = {
+  title: string;
+  category: string,
+  startTime: string;
+  duration: number;
+  error?: TestError;
+  steps: JsonTestStep[];
+  log?: string[];
+};
+
+class RawReporter {
+  private config!: FullConfig;
+  private suite!: Suite;
+
+  onBegin(config: FullConfig, suite: Suite) {
+    this.config = config;
+    this.suite = suite;
+  }
+
+  async onEnd() {
+    const projectSuites = this.suite.suites;
+    for (const suite of projectSuites) {
+      const project = (suite as any)._projectConfig as FullProject;
+      assert(project, 'Internal Error: Invalid project structure');
+      const reportFolder = path.join(project.outputDir, 'report');
+      fs.mkdirSync(reportFolder, { recursive: true });
+      let reportFile: string | undefined;
+      for (let i = 0; i < 10; ++i) {
+        reportFile = path.join(reportFolder, sanitizeForFilePath(project.name || 'project') + (i ? '-' + i : '') + '.report');
+        try {
+          if (fs.existsSync(reportFile))
+            continue;
+        } catch (e) {
+        }
+        break;
+      }
+      if (!reportFile)
+        throw new Error('Internal error, could not create report file');
+      const report: JsonReport = {
+        config: this.config,
+        project: {
+          metadata: project.metadata,
+          name: project.name,
+          outputDir: toPosixPath(project.outputDir),
+          repeatEach: project.repeatEach,
+          retries: project.retries,
+          testDir: toPosixPath(project.testDir),
+          testIgnore: serializePatterns(project.testIgnore),
+          testMatch: serializePatterns(project.testMatch),
+          timeout: project.timeout,
+        },
+        suites: suite.suites.map(s => this._serializeSuite(s, reportFolder))
+      };
+      fs.writeFileSync(reportFile, JSON.stringify(report, undefined, 2));
+    }
+  }
+
+  private _serializeSuite(suite: Suite, reportFolder: string): JsonSuite {
+    return {
+      title: suite.title,
+      location: suite.location,
+      suites: suite.suites.map(s => this._serializeSuite(s, reportFolder)),
+      tests: suite.tests.map(t => this._serializeTest(t, reportFolder)),
+    };
+  }
+
+  private _serializeTest(test: TestCase, reportFolder: string): JsonTestCase {
+    const testId = calculateSha1(test.titlePath().join('|'));
+    return {
+      testId,
+      title: test.title,
+      location: test.location,
+      expectedStatus: test.expectedStatus,
+      timeout: test.timeout,
+      annotations: test.annotations,
+      retries: test.retries,
+      ok: test.ok(),
+      outcome: test.outcome(),
+      results: test.results.map(r => this._serializeResult(testId, test, r, reportFolder)),
+    };
+  }
+
+  private _serializeResult(testId: string, test: TestCase, result: TestResult, reportFolder: string): JsonTestResult {
+    return {
+      retry: result.retry,
+      workerIndex: result.workerIndex,
+      startTime: result.startTime.toISOString(),
+      duration: result.duration,
+      status: result.status,
+      error: result.error,
+      attachments: this._createAttachments(reportFolder, testId, result),
+      steps: this._serializeSteps(test, result.steps)
+    };
+  }
+
+  private _serializeSteps(test: TestCase, steps: TestStep[]): JsonTestStep[] {
+    return steps.map(step => {
+      return {
+        title: step.title,
+        category: step.category,
+        startTime: step.startTime.toISOString(),
+        duration: step.duration,
+        error: step.error,
+        steps: this._serializeSteps(test, step.steps),
+        log: step.data.log || undefined,
+      };
+    });
+  }
+
+  private _createAttachments(reportFolder: string, testId: string, result: TestResult): JsonAttachment[] {
+    const attachments: JsonAttachment[] = [];
+    for (const attachment of result.attachments.filter(a => !a.path)) {
+      const sha1 = calculateSha1(attachment.body!);
+      const file = path.join(reportFolder, sha1);
+      try {
+        fs.writeFileSync(path.join(reportFolder, sha1), attachment.body);
+        attachments.push({
+          name: attachment.name,
+          contentType: attachment.contentType,
+          path: toPosixPath(file)
+        });
+      } catch (e) {
+      }
+    }
+    for (const attachment of result.attachments.filter(a => a.path))
+      attachments.push(attachment as JsonAttachment);
+
+    if (result.stdout.length)
+      attachments.push(this._stdioAttachment(reportFolder, testId, result, 'stdout'));
+    if (result.stderr.length)
+      attachments.push(this._stdioAttachment(reportFolder, testId, result, 'stderr'));
+    return attachments;
+  }
+
+  private _stdioAttachment(reportFolder: string, testId: string, result: TestResult, type: 'stdout' | 'stderr'): JsonAttachment {
+    const file = `${testId}.${result.retry}.${type}`;
+    const fileName = path.join(reportFolder, file);
+    for (const chunk of type === 'stdout' ? result.stdout : result.stderr) {
+      if (typeof chunk === 'string')
+        fs.appendFileSync(fileName, chunk + '\n');
+      else
+        fs.appendFileSync(fileName, chunk);
+    }
+    return {
+      name: type,
+      contentType: 'application/octet-stream',
+      path: toPosixPath(fileName)
+    };
+  }
+}
+
+export default RawReporter;

src/test/runner.ts

@@ -32,9 +32,10 @@ import ListReporter from './reporters/list';
 import JSONReporter from './reporters/json';
 import JUnitReporter from './reporters/junit';
 import EmptyReporter from './reporters/empty';
+import RawReporter from './reporters/raw';
 import { ProjectImpl } from './project';
 import { Minimatch } from 'minimatch';
-import { Config, FullConfig } from './types';
+import { FullConfig } from './types';
 import { WebServer } from './webServer';
 import { raceAgainstDeadline } from '../utils/async';
 
@@ -59,12 +60,11 @@ export class Runner {
   private _reporter!: Reporter;
   private _didBegin = false;
 
-  constructor(defaultConfig: Config, configOverrides: Config) {
-    this._loader = new Loader(defaultConfig, configOverrides);
+  constructor(loader: Loader) {
+    this._loader = loader;
   }
 
   private async _createReporter(list: boolean) {
-    const reporters: Reporter[] = [];
     const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = {
       dot: list ? ListModeReporter : DotReporter,
       line: list ? ListModeReporter : LineReporter,
@@ -73,6 +73,7 @@ export class Runner {
       junit: JUnitReporter,
       null: EmptyReporter,
     };
+    const reporters: Reporter[] = [ new RawReporter() ];
     for (const r of this._loader.fullConfig().reporter) {
       const [name, arg] = r;
       if (name in defaultReporters) {
@@ -85,14 +86,6 @@ export class Runner {
     return new Multiplexer(reporters);
   }
 
-  loadConfigFile(file: string): Promise<Config> {
-    return this._loader.loadConfigFile(file);
-  }
-
-  loadEmptyConfig(rootDir: string) {
-    this._loader.loadEmptyConfig(rootDir);
-  }
-
   async run(list: boolean, filePatternFilters: FilePatternFilter[], projectNames?: string[]): Promise<RunResultStatus> {
     this._reporter = await this._createReporter(list);
     const config = this._loader.fullConfig();
@@ -222,6 +215,7 @@ export class Runner {
       const rootSuite = new Suite('');
       for (const project of projects) {
         const projectSuite = new Suite(project.config.name);
+        projectSuite._projectConfig = project.config;
         rootSuite._addSuite(projectSuite);
         for (const file of files.get(project)!) {
           const fileSuite = fileSuites.get(file);

src/test/test.ts

@@ -18,6 +18,7 @@ import type { FixturePool } from './fixtures';
 import * as reporterTypes from '../../types/testReporter';
 import type { TestTypeImpl } from './testType';
 import { Annotations, FixturesWithLocation, Location } from './types';
+import { FullProject } from '../../types/test';
 
 class Base {
   title: string;
@@ -57,6 +58,7 @@ export class Suite extends Base implements reporterTypes.Suite {
   _annotations: Annotations = [];
   _modifiers: Modifier[] = [];
   _parallelMode: 'default' | 'serial' | 'parallel' = 'default';
+  _projectConfig: FullProject | undefined;
 
   _addTest(test: TestCase) {
     test.parent = this;

src/web/htmlReport2/htmlReport.css

@@ -0,0 +1,186 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+.suite-tree-column {
+  line-height: 18px;
+  flex: auto;
+  overflow: auto;
+  color: #616161;
+  background-color: #f3f3f3;
+}
+
+.tree-item-title {
+  padding: 8px 0;
+  cursor: pointer;
+}
+
+.tree-item-body {
+  min-height: 18px;
+}
+
+.suite-tree-column .tree-item-title:not(.selected):hover {
+  background-color: #e8e8e8;
+}
+
+.suite-tree-column .tree-item-title.selected {
+  background-color: #0060c0;
+  color: white;
+}
+
+.suite-tree-column .tree-item-title.selected * {
+  color: white !important;
+}
+
+.error-message {
+  white-space: pre;
+  font-family: monospace;
+  background: #000;
+  color: white;
+  padding: 5px;
+  overflow: auto;
+  margin: 20px 0;
+  flex: auto;
+}
+
+.status-icon {
+  padding-right: 3px;
+}
+
+.codicon {
+  padding-right: 3px;
+}
+
+.codicon-clock.status-icon,
+.codicon-error.status-icon {
+  color: red;
+}
+
+.codicon-alert.status-icon {
+  color: orange;
+}
+
+.codicon-circle-filled.status-icon {
+  color: green;
+}
+
+.test-result {
+  flex: auto;
+  display: flex;
+  flex-direction: column;
+  padding-right: 8px;
+}
+
+.test-overview-title {
+  padding: 10px 0;
+  font-size: 18px;
+  flex: none;
+}
+
+.image-preview img {
+  max-width: 500px;
+  max-height: 500px;
+}
+
+.image-preview {
+  position: relative;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  width: 550px;
+  height: 550px;
+}
+
+.test-result .tabbed-pane .tab-content {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.attachment-body {
+  white-space: pre-wrap;
+  font-family: monospace;
+  background-color: #dadada;
+  border: 1px solid #ccc;
+  margin-left: 24px;
+}
+
+.test-result .tree-item-title:not(.selected):hover {
+  background-color: #e8e8e8;
+}
+
+.test-result .tree-item-title.selected {
+  background-color: #0060c0;
+  color: white;
+}
+
+.test-result .tree-item-title.selected * {
+  color: white !important;
+}
+
+.suite-tree-column .tab-strip,
+.test-case-column .tab-strip {
+  border: none;
+  box-shadow: none;
+  background-color: transparent;
+}
+
+.suite-tree-column .tab-element,
+.test-case-column .tab-element {
+  border: none;
+  text-transform: uppercase;
+  font-weight: bold;
+  font-size: 11px;
+  color: #aaa;
+}
+
+.suite-tree-column .tab-element.selected,
+.test-case-column .tab-element.selected {
+  color: #555;
+}
+
+.test-case-title {
+  flex: none;
+  display: flex;
+  align-items: center;
+  padding: 10px;
+  font-size: 18px;
+  cursor: pointer;
+}
+
+.test-case-location {
+  flex: none;
+  display: flex;
+  align-items: center;
+  padding: 0 10px 10px;
+  color: var(--blue);
+  text-decoration: underline;
+  cursor: pointer;
+}
+
+.test-details-column {
+  overflow-y: auto;
+}
+
+.step-log {
+  line-height: 20px;
+  white-space: pre;
+  padding: 8px;
+}
+
+.tree-text {
+  overflow: hidden;
+  text-overflow: ellipsis;
+}

src/web/htmlReport2/htmlReport.tsx

@@ -0,0 +1,204 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import './htmlReport.css';
+import * as React from 'react';
+import { SplitView } from '../components/splitView';
+import { TreeItem } from '../components/treeItem';
+import { TabbedPane } from '../traceViewer/ui/tabbedPane';
+import { msToString } from '../uiUtils';
+import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location } from '../../test/html/types';
+
+type Filter = 'Failing' | 'All';
+
+export const Report: React.FC = () => {
+  const [report, setReport] = React.useState<ProjectTreeItem[]>([]);
+  const [fetchError, setFetchError] = React.useState<string | undefined>();
+  const [testId, setTestId] = React.useState<string | undefined>();
+
+  React.useEffect(() => {
+    (async () => {
+      try {
+        const result = await fetch('data/projects.json', { cache: 'no-cache' });
+        const json = (await result.json()) as ProjectTreeItem[];
+        setReport(json);
+      } catch (e) {
+        setFetchError(e.message);
+      }
+    })();
+  }, []);
+  const [filter, setFilter] = React.useState<Filter>('Failing');
+
+  return <div className='hbox'>
+    <SplitView sidebarSize={300} orientation='horizontal' sidebarIsFirst={true}>
+      <TestCaseView testId={testId}></TestCaseView>
+      <div className='suite-tree-column'>
+        <div className='tab-strip'>{
+          (['Failing', 'All'] as Filter[]).map(item => {
+            const selected = item === filter;
+            return <div key={item} className={'tab-element' + (selected ? ' selected' : '')} onClick={e => {
+              setFilter(item);
+            }}>{item}</div>;
+          })
+        }</div>
+        {!fetchError && filter === 'All' && report?.map((project, i) => <ProjectTreeItemView key={i} project={project} setTestId={setTestId} testId={testId}></ProjectTreeItemView>)}
+        {!fetchError && filter === 'Failing' && report?.map((project, i) => <ProjectTreeItemView key={i} project={project} setTestId={setTestId} testId={testId} failingOnly={true}></ProjectTreeItemView>)}
+      </div>
+    </SplitView>
+  </div>;
+};
+
+const ProjectTreeItemView: React.FC<{
+  project: ProjectTreeItem;
+  testId?: string,
+  setTestId: (id: string) => void;
+  failingOnly?: boolean;
+}> = ({ project, testId, setTestId, failingOnly }) => {
+  return <TreeItem title={<div className='hbox'>
+    {statusIconForFailedTests(project.failedTests)}<div className='tree-text'>{project.name || 'Project'}</div>
+  </div>
+  } loadChildren={() => {
+    return project.suits.map((s, i) => <SuiteTreeItemView key={i} suite={s} setTestId={setTestId} testId={testId} depth={1} showFileName={true}></SuiteTreeItemView>) || [];
+  }} depth={0} expandByDefault={true}></TreeItem>;
+};
+
+const SuiteTreeItemView: React.FC<{
+  suite: SuiteTreeItem,
+  testId?: string,
+  setTestId: (id: string) => void;
+  depth: number,
+  showFileName: boolean,
+}> = ({ suite, testId, setTestId, showFileName, depth }) => {
+  const location = renderLocation(suite.location, showFileName);
+  return <TreeItem title={<div className='hbox'>
+    {statusIconForFailedTests(suite.failedTests)}<div className='tree-text'>{suite.title}</div>
+    {!!suite.location?.line && location && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{location}</div>}
+  </div>
+  } loadChildren={() => {
+    const suiteChildren = suite.suites.map((s, i) => <SuiteTreeItemView key={i} suite={s} setTestId={setTestId} testId={testId} depth={depth + 1} showFileName={false}></SuiteTreeItemView>) || [];
+    const suiteCount = suite.suites.length;
+    const testChildren = suite.tests.map((t, i) => <TestTreeItemView key={i + suiteCount} test={t} setTestId={setTestId} testId={testId} showFileName={false} depth={depth + 1}></TestTreeItemView>) || [];
+    return [...suiteChildren, ...testChildren];
+  }} depth={depth}></TreeItem>;
+};
+
+const TestTreeItemView: React.FC<{
+  test: TestTreeItem,
+  showFileName: boolean,
+  testId?: string,
+  setTestId: (id: string) => void;
+  depth: number,
+}> = ({ test, testId, setTestId, showFileName, depth }) => {
+  const fileName = test.location.file;
+  const name = fileName.substring(fileName.lastIndexOf('/') + 1);
+  return <TreeItem title={<div className='hbox'>
+    {statusIcon(test.outcome)}<div className='tree-text'>{test.title}</div>
+    {showFileName && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{name}:{test.location.line}</div>}
+    {!showFileName && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{msToString(test.duration)}</div>}
+  </div>
+  } selected={test.testId === testId} depth={depth} onClick={() => setTestId(test.testId)}></TreeItem>;
+};
+
+const TestCaseView: React.FC<{
+  testId: string | undefined,
+}> = ({ testId }) => {
+  const [test, setTest] = React.useState<TestCase | undefined>();
+
+  React.useEffect(() => {
+    (async () => {
+      if (!testId)
+        return;
+      try {
+        const result = await fetch(`data/${testId}.json`, { cache: 'no-cache' });
+        const json = (await result.json()) as TestCase;
+        setTest(json);
+      } catch (e) {
+      }
+    })();
+  });
+
+  const [selectedResultIndex, setSelectedResultIndex] = React.useState(0);
+  return <SplitView sidebarSize={500} orientation='horizontal' sidebarIsFirst={true}>
+    <div className='test-details-column vbox'>
+    </div>
+    <div className='test-case-column vbox'>
+      { test && <div className='test-case-title'>{test?.title}</div> }
+      { test && <div className='test-case-location'>{renderLocation(test.location, true)}</div> }
+      { test && <TabbedPane tabs={
+        test?.results.map((result, index) => ({
+          id: String(index),
+          title: <div style={{ display: 'flex', alignItems: 'center' }}>{statusIcon(result.status)} {retryLabel(index)}</div>,
+          render: () => <TestResultView test={test} result={result}></TestResultView>
+        })) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />}
+    </div>
+  </SplitView>;
+};
+
+const TestResultView: React.FC<{
+  test: TestCase,
+  result: TestResult,
+}> = ({ test, result }) => {
+  return <div className='test-result'>
+    {result.steps.map((step, i) => <StepTreeItem key={i} step={step} depth={0}></StepTreeItem>)}
+  </div>;
+};
+
+const StepTreeItem: React.FC<{
+  step: TestStep;
+  depth: number,
+}> = ({ step, depth }) => {
+  return <TreeItem title={<div style={{ display: 'flex', alignItems: 'center', flex: 'auto' }}>
+    {statusIcon(step.error ? 'failed' : 'passed')}
+    <span style={{ whiteSpace: 'pre' }}>{step.title}</span>
+    <div style={{ flex: 'auto' }}></div>
+    <div>{msToString(step.duration)}</div>
+  </div>} loadChildren={step.steps.length ? () => {
+    return step.steps.map((s, i) => <StepTreeItem key={i} step={s} depth={depth + 1}></StepTreeItem>);
+  } : undefined} depth={depth}></TreeItem>;
+};
+
+function statusIconForFailedTests(failedTests: number) {
+  return failedTests ? statusIcon('failed') : statusIcon('passed');
+}
+
+function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expected' | 'unexpected' | 'flaky'): JSX.Element {
+  switch (status) {
+    case 'failed':
+    case 'unexpected':
+      return <span className={'codicon codicon-error status-icon'}></span>;
+    case 'passed':
+    case 'expected':
+      return <span className={'codicon codicon-circle-filled status-icon'}></span>;
+    case 'timedOut':
+      return <span className={'codicon codicon-clock status-icon'}></span>;
+    case 'flaky':
+      return <span className={'codicon codicon-alert status-icon'}></span>;
+    case 'skipped':
+      return <span className={'codicon codicon-tag status-icon'}></span>;
+  }
+}
+
+function renderLocation(location: Location | undefined, showFileName: boolean) {
+  if (!location)
+    return '';
+  return (showFileName ? location.file : '') + ':' + location.line;
+}
+
+function retryLabel(index: number) {
+  if (!index)
+    return 'Run';
+  return `Retry #${index}`;
+}

src/web/htmlReport2/index.html

@@ -0,0 +1,27 @@
+<!--
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Playwright Test Report</title>
+  </head>
+  <body>
+    <div id=root></div>
+  </body>
+</html>

src/web/htmlReport2/index.tsx

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import '../third_party/vscode/codicon.css';
+import * as React from 'react';
+import * as ReactDOM from 'react-dom';
+import { applyTheme } from '../theme';
+import '../common.css';
+import { Report } from './htmlReport';
+
+(async () => {
+  applyTheme();
+  ReactDOM.render(<Report />, document.querySelector('#root'));
+})();

src/web/htmlReport2/webpack.config.js

@@ -0,0 +1,50 @@
+const path = require('path');
+const HtmlWebPackPlugin = require('html-webpack-plugin');
+
+const mode = process.env.NODE_ENV === 'production' ? 'production' : 'development';
+
+module.exports = {
+  mode,
+  entry: {
+    app: path.join(__dirname, 'index.tsx'),
+  },
+  resolve: {
+    extensions: ['.ts', '.js', '.tsx', '.jsx']
+  },
+  devtool: mode === 'production' ? false : 'source-map',
+  output: {
+    globalObject: 'self',
+    filename: '[name].bundle.js',
+    path: path.resolve(__dirname, '../../../lib/web/htmlReport2')
+  },
+  module: {
+    rules: [
+      {
+        test: /\.(j|t)sx?$/,
+        loader: 'babel-loader',
+        options: {
+          presets: [
+            "@babel/preset-typescript",
+            "@babel/preset-react"
+          ]
+        },
+        exclude: /node_modules/
+      },
+      {
+        test: /\.css$/,
+        use: ['style-loader', 'css-loader']
+      },
+      {
+        test: /\.ttf$/,
+        use: ['file-loader']
+      }
+    ]
+  },
+  plugins: [
+    new HtmlWebPackPlugin({
+      title: 'Playwright Test Report',
+      template: path.join(__dirname, 'index.html'),
+      inject: true,
+    })
+  ]
+};

tests/playwright-test/playwright.artifacts.spec.ts

@@ -155,6 +155,8 @@ test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => {
     'artifacts-two-contexts-failing',
     '  test-failed-1.png',
     '  test-failed-2.png',
+    'report',
+    '  project.report',
     'report.json',
   ]);
 });
@@ -182,6 +184,8 @@ test('should work with screenshot: only-on-failure', async ({ runInlineTest }, t
     'artifacts-two-contexts-failing',
     '  test-failed-1.png',
     '  test-failed-2.png',
+    'report',
+    '  project.report',
     'report.json',
   ]);
 });
@@ -220,6 +224,8 @@ test('should work with trace: on', async ({ runInlineTest }, testInfo) => {
     'artifacts-two-contexts-failing',
     '  trace-1.zip',
     '  trace.zip',
+    'report',
+    '  project.report',
     'report.json',
   ]);
 });
@@ -247,6 +253,8 @@ test('should work with trace: retain-on-failure', async ({ runInlineTest }, test
     'artifacts-two-contexts-failing',
     '  trace-1.zip',
     '  trace.zip',
+    'report',
+    '  project.report',
     'report.json',
   ]);
 });
@@ -274,6 +282,8 @@ test('should work with trace: on-first-retry', async ({ runInlineTest }, testInf
     'artifacts-two-contexts-failing-retry1',
     '  trace-1.zip',
     '  trace.zip',
+    'report',
+    '  project.report',
     'report.json',
   ]);
 });
@@ -314,6 +324,8 @@ test('should stop tracing with trace: on-first-retry, when not retrying', async
   expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
     'a-shared-flaky-retry1',
     '  trace.zip',
+    'report',
+    '  project.report',
     'report.json',
   ]);
 });

tests/playwright-test/raw-reporter.spec.ts

@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import { test, expect } from './playwright-test-fixtures';
+
+test('should generate raw report', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('passes', async ({ page }, testInfo) => {});
+    `,
+  }, { usesCustomOutputDir: true });
+  const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8'));
+  expect(json.config).toBeTruthy();
+  expect(json.project).toBeTruthy();
+  expect(result.exitCode).toBe(0);
+});
+
+test('should use project name', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [{
+          name: 'project-name',
+          outputDir: 'output'
+        }]
+      }
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('passes', async ({ page }, testInfo) => {});
+    `,
+  }, { usesCustomOutputDir: true });
+  const json = JSON.parse(fs.readFileSync(testInfo.outputPath('output', 'report', 'project-name.report'), 'utf-8'));
+  expect(json.project.name).toBe('project-name');
+  expect(result.exitCode).toBe(0);
+});
+
+test('should save stdio', async ({ runInlineTest }, testInfo) => {
+  await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('passes', async ({ page }, testInfo) => {
+        console.log('STDOUT');
+        console.error('STDERR');
+      });
+    `,
+  }, { usesCustomOutputDir: true });
+  const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8'));
+  const result = json.suites[0].tests[0].results[0];
+  expect(result.attachments[0].name).toBe('stdout');
+  expect(result.attachments[1].name).toBe('stderr');
+  const path1 = result.attachments[0].path;
+  expect(fs.readFileSync(path1, 'utf-8')).toContain('STDOUT');
+  const path2 = result.attachments[1].path;
+  expect(fs.readFileSync(path2, 'utf-8')).toContain('STDERR');
+});
+
+test('should save attachments', async ({ runInlineTest }, testInfo) => {
+  await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('passes', async ({ page }, testInfo) => {
+        testInfo.attachments.push({
+          name: 'binary',
+          contentType: 'application/octet-stream',
+          body: Buffer.from([1,2,3])
+        });
+        testInfo.attachments.push({
+          name: 'text',
+          contentType: 'text/plain',
+          path: 'dummy-path'
+        });
+      });
+    `,
+  }, { usesCustomOutputDir: true });
+  const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8'));
+  const result = json.suites[0].tests[0].results[0];
+  expect(result.attachments[0].name).toBe('binary');
+  expect(result.attachments[1].name).toBe('text');
+  const path1 = result.attachments[0].path;
+  expect(fs.readFileSync(path1)).toEqual(Buffer.from([1,2,3]));
+  const path2 = result.attachments[1].path;
+  expect(path2).toBe('dummy-path');
+});
+
+test('dupe project names', async ({ runInlineTest }, testInfo) => {
+  await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'project-name' },
+          { name: 'project-name' },
+          { name: 'project-name' },
+        ]
+      }
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('passes', async ({ page }, testInfo) => {});
+    `,
+  }, { usesCustomOutputDir: true });
+  const files = fs.readdirSync(testInfo.outputPath('test-results', 'report'));
+  expect(new Set(files)).toEqual(new Set(['project-name.report', 'project-name-1.report', 'project-name-2.report']));
+});

utils/build/build.js

@@ -115,6 +115,7 @@ const webPackFiles = [
   'src/web/traceViewer/webpack.config.js',
   'src/web/recorder/webpack.config.js',
   'src/web/htmlReport/webpack.config.js',
+  'src/web/htmlReport2/webpack.config.js',
 ];
 for (const file of webPackFiles) {
   steps.push({

utils/check_deps.js

@@ -195,6 +195,7 @@ DEPS['src/test/'] = ['src/test/**', 'src/utils/utils.ts', 'src/utils/**'];
 
 // HTML report
 DEPS['src/web/htmlReport/'] = ['src/test/**', 'src/web/'];
+DEPS['src/web/htmlReport2/'] = ['src/test/**', 'src/web/'];
 
 
 checkDeps().catch(e => {