diff --git a/.gitignore b/.gitignore index ed0ce8abca..52ac3fd320 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ nohup.out .tmp allure* playwright-report +/demo/ diff --git a/src/cli/cli.ts b/src/cli/cli.ts index e74a1e99f1..03108a4f49 100755 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -237,6 +237,8 @@ if (!process.env.PW_CLI_TARGET_LANG) { if (playwrightTestPackagePath) { require(playwrightTestPackagePath).addTestCommand(program); + if (process.env.PW_EXPERIMENTAL) + require(playwrightTestPackagePath).addGenerateHtmlCommand(program); } else { const command = program.command('test').allowUnknownOption(true); command.description('Run tests with Playwright Test. Available in @playwright/test package.'); diff --git a/src/test/cli.ts b/src/test/cli.ts index 86986b599d..e8efb8ec57 100644 --- a/src/test/cli.ts +++ b/src/test/cli.ts @@ -23,6 +23,8 @@ import type { Config } from './types'; import { Runner, builtInReporters, BuiltInReporter } from './runner'; import { stopProfiling, startProfiling } from './profiler'; import { FilePatternFilter } from './util'; +import { Loader } from './loader'; +import { HtmlBuilder } from './html/htmlBuilder'; const defaultTimeout = 30000; const defaultReporter: BuiltInReporter = process.env.CI ? 'dot' : 'list'; @@ -81,9 +83,31 @@ export function addTestCommand(program: commander.CommanderStatic) { }); } -async function runTests(args: string[], opts: { [key: string]: any }) { - await startProfiling(); +export function addGenerateHtmlCommand(program: commander.CommanderStatic) { + const command = program.command('generate-html'); + command.description('Generate HTML report'); + command.option('-c, --config ', `Configuration file, or a test directory with optional "${tsConfig}"/"${jsConfig}"`); + command.option('--output ', `Folder for output artifacts (default: "playwright-report")`, 'playwright-report'); + command.action(async opts => { + const loader = await createLoader(opts); + const outputFolders = new Set(loader.projects().map(p => p.config.outputDir)); + const reportFiles = new Set(); + for (const outputFolder of outputFolders) { + const reportFolder = path.join(outputFolder, 'report'); + const files = fs.readdirSync(reportFolder).filter(f => f.endsWith('.report')); + for (const file of files) + reportFiles.add(path.join(reportFolder, file)); + } + new HtmlBuilder([...reportFiles], opts.output); + }).on('--help', () => { + console.log(''); + console.log('Examples:'); + console.log(''); + console.log(' $ generate-report'); + }); +} +async function createLoader(opts: { [key: string]: any }): Promise { if (opts.browser) { const browserOpt = opts.browser.toLowerCase(); if (!['all', 'chromium', 'firefox', 'webkit'].includes(browserOpt)) @@ -100,13 +124,13 @@ async function runTests(args: string[], opts: { [key: string]: any }) { const overrides = overridesFromOptions(opts); if (opts.headed) overrides.use = { headless: false }; - const runner = new Runner(defaultConfig, overrides); + const loader = new Loader(defaultConfig, overrides); async function loadConfig(configFile: string) { if (fs.existsSync(configFile)) { if (process.stdout.isTTY) console.log(`Using config at ` + configFile); - const loadedConfig = await runner.loadConfigFile(configFile); + const loadedConfig = await loader.loadConfigFile(configFile); if (('projects' in loadedConfig) && opts.browser) throw new Error(`Cannot use --browser option when configuration file defines projects. Specify browserName in the projects instead.`); return true; @@ -131,7 +155,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) { // When passed a directory, look for a config file inside. if (!await loadConfigFromDirectory(configFile)) { // If there is no config, assume this as a root testing directory. - runner.loadEmptyConfig(configFile); + loader.loadEmptyConfig(configFile); } } else { // When passed a file, it must be a config file. @@ -140,9 +164,15 @@ async function runTests(args: string[], opts: { [key: string]: any }) { } else if (!await loadConfigFromDirectory(process.cwd())) { // No --config option, let's look for the config file in the current directory. // If not, scan the world. - runner.loadEmptyConfig(process.cwd()); + loader.loadEmptyConfig(process.cwd()); } + return loader; +} +async function runTests(args: string[], opts: { [key: string]: any }) { + await startProfiling(); + + const loader = await createLoader(opts); const filePatternFilters: FilePatternFilter[] = args.map(arg => { const match = /^(.*):(\d+)$/.exec(arg); return { @@ -150,6 +180,8 @@ async function runTests(args: string[], opts: { [key: string]: any }) { line: match ? parseInt(match[2], 10) : null, }; }); + + const runner = new Runner(loader); const result = await runner.run(!!opts.list, filePatternFilters, opts.project || undefined); await stopProfiling(undefined); diff --git a/src/test/html/htmlBuilder.ts b/src/test/html/htmlBuilder.ts new file mode 100644 index 0000000000..920c0a4393 --- /dev/null +++ b/src/test/html/htmlBuilder.ts @@ -0,0 +1,101 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; +import { ProjectTreeItem, SuiteTreeItem, TestTreeItem, TestCase, TestResult, TestStep } from './types'; +import { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../reporters/raw'; + +export class HtmlBuilder { + private _reportFolder: string; + private _tests = new Map(); + + constructor(rawReports: string[], outputDir: string) { + this._reportFolder = path.resolve(process.cwd(), outputDir); + const dataFolder = path.join(this._reportFolder, 'data'); + fs.mkdirSync(dataFolder, { recursive: true }); + const appFolder = path.join(__dirname, '..', '..', 'web', 'htmlReport2'); + for (const file of fs.readdirSync(appFolder)) + fs.copyFileSync(path.join(appFolder, file), path.join(this._reportFolder, file)); + const projects: ProjectTreeItem[] = rawReports.map(rawReport => { + const json = JSON.parse(fs.readFileSync(rawReport, 'utf-8')) as JsonReport; + const suits = json.suites.map(s => this._createSuiteTreeItem(s)); + return { + name: json.project.name, + suits, + failedTests: suits.reduce((a, s) => a + s.failedTests, 0) + }; + }); + fs.writeFileSync(path.join(dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2)); + + for (const [testId, test] of this._tests) { + const testCase: TestCase = { + testId: test.testId, + title: test.title, + location: test.location, + results: test.results.map(r => this._createTestResult(r)) + }; + fs.writeFileSync(path.join(dataFolder, testId + '.json'), JSON.stringify(testCase, undefined, 2)); + } + } + + private _createSuiteTreeItem(suite: JsonSuite): SuiteTreeItem { + const suites = suite.suites.map(s => this._createSuiteTreeItem(s)); + const tests = suite.tests.map(t => this._createTestTreeItem(t)); + return { + title: suite.title, + location: suite.location, + duration: suites.reduce((a, s) => a + s.duration, 0) + tests.reduce((a, t) => a + t.duration, 0), + failedTests: suites.reduce((a, s) => a + s.failedTests, 0) + tests.reduce((a, t) => t.outcome === 'unexpected' || t.outcome === 'flaky' ? a + 1 : a, 0), + suites, + tests + }; + } + + private _createTestTreeItem(test: JsonTestCase): TestTreeItem { + const duration = test.results.reduce((a, r) => a + r.duration, 0); + this._tests.set(test.testId, test); + return { + testId: test.testId, + location: test.location, + title: test.title, + duration, + outcome: test.outcome + }; + } + + private _createTestResult(result: JsonTestResult): TestResult { + return { + duration: result.duration, + startTime: result.startTime, + retry: result.retry, + steps: result.steps.map(s => this._createTestStep(s)), + error: result.error, + status: result.status, + }; + } + + private _createTestStep(step: JsonTestStep): TestStep { + return { + title: step.title, + startTime: step.startTime, + duration: step.duration, + steps: step.steps.map(s => this._createTestStep(s)), + log: step.log, + error: step.error + }; + } +} diff --git a/src/test/html/types.ts b/src/test/html/types.ts new file mode 100644 index 0000000000..e130490190 --- /dev/null +++ b/src/test/html/types.ts @@ -0,0 +1,75 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type Location = { + file: string; + line: number; + column: number; +}; + +export type ProjectTreeItem = { + name: string; + suits: SuiteTreeItem[]; + failedTests: number; +}; + +export type SuiteTreeItem = { + title: string; + location?: Location; + duration: number; + suites: SuiteTreeItem[]; + tests: TestTreeItem[]; + failedTests: number; +}; + +export type TestTreeItem = { + testId: string, + title: string; + location: Location; + duration: number; + outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; +}; + +export type TestCase = { + testId: string, + title: string; + location: Location; + results: TestResult[]; +}; + +export interface TestError { + message?: string; + stack?: string; + value?: string; +} + +export type TestResult = { + retry: number; + startTime: string; + duration: number; + steps: TestStep[]; + error?: TestError; + status: 'passed' | 'failed' | 'timedOut' | 'skipped'; +}; + +export type TestStep = { + title: string; + startTime: string; + duration: number; + log?: string[]; + error?: TestError; + steps: TestStep[]; +}; diff --git a/src/test/reporters/raw.ts b/src/test/reporters/raw.ts new file mode 100644 index 0000000000..139b0cb100 --- /dev/null +++ b/src/test/reporters/raw.ts @@ -0,0 +1,244 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; +import { FullProject } from '../../../types/test'; +import { FullConfig, Location, Suite, TestCase, TestError, TestResult, TestStatus, TestStep } from '../../../types/testReporter'; +import { assert, calculateSha1 } from '../../utils/utils'; +import { sanitizeForFilePath } from '../util'; +import { serializePatterns, toPosixPath } from './json'; + +export type JsonStats = { expected: number, unexpected: number, flaky: number, skipped: number }; +export type JsonLocation = Location; +export type JsonStackFrame = { file: string, line: number, column: number }; + +export type JsonReport = { + config: JsonConfig, + project: JsonProject, + suites: JsonSuite[], +}; + +export type JsonConfig = Omit; + +export type JsonProject = { + metadata: any, + name: string, + outputDir: string, + repeatEach: number, + retries: number, + testDir: string, + testIgnore: string[], + testMatch: string[], + timeout: number, +}; + +export type JsonSuite = { + title: string; + location?: JsonLocation; + suites: JsonSuite[]; + tests: JsonTestCase[]; +}; + +export type JsonTestCase = { + testId: string; + title: string; + location: JsonLocation; + expectedStatus: TestStatus; + timeout: number; + annotations: { type: string, description?: string }[]; + retries: number; + results: JsonTestResult[]; + ok: boolean; + outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; +}; + +export type TestAttachment = { + name: string; + path?: string; + body?: Buffer; + contentType: string; +}; + +export type JsonAttachment = { + name: string; + path: string; + contentType: string; +}; + +export type JsonTestResult = { + retry: number; + workerIndex: number; + startTime: string; + duration: number; + status: TestStatus; + error?: TestError; + attachments: JsonAttachment[]; + steps: JsonTestStep[]; +}; + +export type JsonTestStep = { + title: string; + category: string, + startTime: string; + duration: number; + error?: TestError; + steps: JsonTestStep[]; + log?: string[]; +}; + +class RawReporter { + private config!: FullConfig; + private suite!: Suite; + + onBegin(config: FullConfig, suite: Suite) { + this.config = config; + this.suite = suite; + } + + async onEnd() { + const projectSuites = this.suite.suites; + for (const suite of projectSuites) { + const project = (suite as any)._projectConfig as FullProject; + assert(project, 'Internal Error: Invalid project structure'); + const reportFolder = path.join(project.outputDir, 'report'); + fs.mkdirSync(reportFolder, { recursive: true }); + let reportFile: string | undefined; + for (let i = 0; i < 10; ++i) { + reportFile = path.join(reportFolder, sanitizeForFilePath(project.name || 'project') + (i ? '-' + i : '') + '.report'); + try { + if (fs.existsSync(reportFile)) + continue; + } catch (e) { + } + break; + } + if (!reportFile) + throw new Error('Internal error, could not create report file'); + const report: JsonReport = { + config: this.config, + project: { + metadata: project.metadata, + name: project.name, + outputDir: toPosixPath(project.outputDir), + repeatEach: project.repeatEach, + retries: project.retries, + testDir: toPosixPath(project.testDir), + testIgnore: serializePatterns(project.testIgnore), + testMatch: serializePatterns(project.testMatch), + timeout: project.timeout, + }, + suites: suite.suites.map(s => this._serializeSuite(s, reportFolder)) + }; + fs.writeFileSync(reportFile, JSON.stringify(report, undefined, 2)); + } + } + + private _serializeSuite(suite: Suite, reportFolder: string): JsonSuite { + return { + title: suite.title, + location: suite.location, + suites: suite.suites.map(s => this._serializeSuite(s, reportFolder)), + tests: suite.tests.map(t => this._serializeTest(t, reportFolder)), + }; + } + + private _serializeTest(test: TestCase, reportFolder: string): JsonTestCase { + const testId = calculateSha1(test.titlePath().join('|')); + return { + testId, + title: test.title, + location: test.location, + expectedStatus: test.expectedStatus, + timeout: test.timeout, + annotations: test.annotations, + retries: test.retries, + ok: test.ok(), + outcome: test.outcome(), + results: test.results.map(r => this._serializeResult(testId, test, r, reportFolder)), + }; + } + + private _serializeResult(testId: string, test: TestCase, result: TestResult, reportFolder: string): JsonTestResult { + return { + retry: result.retry, + workerIndex: result.workerIndex, + startTime: result.startTime.toISOString(), + duration: result.duration, + status: result.status, + error: result.error, + attachments: this._createAttachments(reportFolder, testId, result), + steps: this._serializeSteps(test, result.steps) + }; + } + + private _serializeSteps(test: TestCase, steps: TestStep[]): JsonTestStep[] { + return steps.map(step => { + return { + title: step.title, + category: step.category, + startTime: step.startTime.toISOString(), + duration: step.duration, + error: step.error, + steps: this._serializeSteps(test, step.steps), + log: step.data.log || undefined, + }; + }); + } + + private _createAttachments(reportFolder: string, testId: string, result: TestResult): JsonAttachment[] { + const attachments: JsonAttachment[] = []; + for (const attachment of result.attachments.filter(a => !a.path)) { + const sha1 = calculateSha1(attachment.body!); + const file = path.join(reportFolder, sha1); + try { + fs.writeFileSync(path.join(reportFolder, sha1), attachment.body); + attachments.push({ + name: attachment.name, + contentType: attachment.contentType, + path: toPosixPath(file) + }); + } catch (e) { + } + } + for (const attachment of result.attachments.filter(a => a.path)) + attachments.push(attachment as JsonAttachment); + + if (result.stdout.length) + attachments.push(this._stdioAttachment(reportFolder, testId, result, 'stdout')); + if (result.stderr.length) + attachments.push(this._stdioAttachment(reportFolder, testId, result, 'stderr')); + return attachments; + } + + private _stdioAttachment(reportFolder: string, testId: string, result: TestResult, type: 'stdout' | 'stderr'): JsonAttachment { + const file = `${testId}.${result.retry}.${type}`; + const fileName = path.join(reportFolder, file); + for (const chunk of type === 'stdout' ? result.stdout : result.stderr) { + if (typeof chunk === 'string') + fs.appendFileSync(fileName, chunk + '\n'); + else + fs.appendFileSync(fileName, chunk); + } + return { + name: type, + contentType: 'application/octet-stream', + path: toPosixPath(fileName) + }; + } +} + +export default RawReporter; diff --git a/src/test/runner.ts b/src/test/runner.ts index 406bb4040f..49ad02bc23 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -32,9 +32,10 @@ import ListReporter from './reporters/list'; import JSONReporter from './reporters/json'; import JUnitReporter from './reporters/junit'; import EmptyReporter from './reporters/empty'; +import RawReporter from './reporters/raw'; import { ProjectImpl } from './project'; import { Minimatch } from 'minimatch'; -import { Config, FullConfig } from './types'; +import { FullConfig } from './types'; import { WebServer } from './webServer'; import { raceAgainstDeadline } from '../utils/async'; @@ -59,12 +60,11 @@ export class Runner { private _reporter!: Reporter; private _didBegin = false; - constructor(defaultConfig: Config, configOverrides: Config) { - this._loader = new Loader(defaultConfig, configOverrides); + constructor(loader: Loader) { + this._loader = loader; } private async _createReporter(list: boolean) { - const reporters: Reporter[] = []; const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = { dot: list ? ListModeReporter : DotReporter, line: list ? ListModeReporter : LineReporter, @@ -73,6 +73,7 @@ export class Runner { junit: JUnitReporter, null: EmptyReporter, }; + const reporters: Reporter[] = [ new RawReporter() ]; for (const r of this._loader.fullConfig().reporter) { const [name, arg] = r; if (name in defaultReporters) { @@ -85,14 +86,6 @@ export class Runner { return new Multiplexer(reporters); } - loadConfigFile(file: string): Promise { - return this._loader.loadConfigFile(file); - } - - loadEmptyConfig(rootDir: string) { - this._loader.loadEmptyConfig(rootDir); - } - async run(list: boolean, filePatternFilters: FilePatternFilter[], projectNames?: string[]): Promise { this._reporter = await this._createReporter(list); const config = this._loader.fullConfig(); @@ -222,6 +215,7 @@ export class Runner { const rootSuite = new Suite(''); for (const project of projects) { const projectSuite = new Suite(project.config.name); + projectSuite._projectConfig = project.config; rootSuite._addSuite(projectSuite); for (const file of files.get(project)!) { const fileSuite = fileSuites.get(file); diff --git a/src/test/test.ts b/src/test/test.ts index 22d34754d3..01a04ae8fe 100644 --- a/src/test/test.ts +++ b/src/test/test.ts @@ -18,6 +18,7 @@ import type { FixturePool } from './fixtures'; import * as reporterTypes from '../../types/testReporter'; import type { TestTypeImpl } from './testType'; import { Annotations, FixturesWithLocation, Location } from './types'; +import { FullProject } from '../../types/test'; class Base { title: string; @@ -57,6 +58,7 @@ export class Suite extends Base implements reporterTypes.Suite { _annotations: Annotations = []; _modifiers: Modifier[] = []; _parallelMode: 'default' | 'serial' | 'parallel' = 'default'; + _projectConfig: FullProject | undefined; _addTest(test: TestCase) { test.parent = this; diff --git a/src/web/htmlReport2/htmlReport.css b/src/web/htmlReport2/htmlReport.css new file mode 100644 index 0000000000..35e1ce4caa --- /dev/null +++ b/src/web/htmlReport2/htmlReport.css @@ -0,0 +1,186 @@ +/* + Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +.suite-tree-column { + line-height: 18px; + flex: auto; + overflow: auto; + color: #616161; + background-color: #f3f3f3; +} + +.tree-item-title { + padding: 8px 0; + cursor: pointer; +} + +.tree-item-body { + min-height: 18px; +} + +.suite-tree-column .tree-item-title:not(.selected):hover { + background-color: #e8e8e8; +} + +.suite-tree-column .tree-item-title.selected { + background-color: #0060c0; + color: white; +} + +.suite-tree-column .tree-item-title.selected * { + color: white !important; +} + +.error-message { + white-space: pre; + font-family: monospace; + background: #000; + color: white; + padding: 5px; + overflow: auto; + margin: 20px 0; + flex: auto; +} + +.status-icon { + padding-right: 3px; +} + +.codicon { + padding-right: 3px; +} + +.codicon-clock.status-icon, +.codicon-error.status-icon { + color: red; +} + +.codicon-alert.status-icon { + color: orange; +} + +.codicon-circle-filled.status-icon { + color: green; +} + +.test-result { + flex: auto; + display: flex; + flex-direction: column; + padding-right: 8px; +} + +.test-overview-title { + padding: 10px 0; + font-size: 18px; + flex: none; +} + +.image-preview img { + max-width: 500px; + max-height: 500px; +} + +.image-preview { + position: relative; + display: flex; + align-items: center; + justify-content: center; + width: 550px; + height: 550px; +} + +.test-result .tabbed-pane .tab-content { + display: flex; + align-items: center; + justify-content: center; +} + +.attachment-body { + white-space: pre-wrap; + font-family: monospace; + background-color: #dadada; + border: 1px solid #ccc; + margin-left: 24px; +} + +.test-result .tree-item-title:not(.selected):hover { + background-color: #e8e8e8; +} + +.test-result .tree-item-title.selected { + background-color: #0060c0; + color: white; +} + +.test-result .tree-item-title.selected * { + color: white !important; +} + +.suite-tree-column .tab-strip, +.test-case-column .tab-strip { + border: none; + box-shadow: none; + background-color: transparent; +} + +.suite-tree-column .tab-element, +.test-case-column .tab-element { + border: none; + text-transform: uppercase; + font-weight: bold; + font-size: 11px; + color: #aaa; +} + +.suite-tree-column .tab-element.selected, +.test-case-column .tab-element.selected { + color: #555; +} + +.test-case-title { + flex: none; + display: flex; + align-items: center; + padding: 10px; + font-size: 18px; + cursor: pointer; +} + +.test-case-location { + flex: none; + display: flex; + align-items: center; + padding: 0 10px 10px; + color: var(--blue); + text-decoration: underline; + cursor: pointer; +} + +.test-details-column { + overflow-y: auto; +} + +.step-log { + line-height: 20px; + white-space: pre; + padding: 8px; +} + +.tree-text { + overflow: hidden; + text-overflow: ellipsis; +} diff --git a/src/web/htmlReport2/htmlReport.tsx b/src/web/htmlReport2/htmlReport.tsx new file mode 100644 index 0000000000..7e5ec79dae --- /dev/null +++ b/src/web/htmlReport2/htmlReport.tsx @@ -0,0 +1,204 @@ +/* + Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import './htmlReport.css'; +import * as React from 'react'; +import { SplitView } from '../components/splitView'; +import { TreeItem } from '../components/treeItem'; +import { TabbedPane } from '../traceViewer/ui/tabbedPane'; +import { msToString } from '../uiUtils'; +import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location } from '../../test/html/types'; + +type Filter = 'Failing' | 'All'; + +export const Report: React.FC = () => { + const [report, setReport] = React.useState([]); + const [fetchError, setFetchError] = React.useState(); + const [testId, setTestId] = React.useState(); + + React.useEffect(() => { + (async () => { + try { + const result = await fetch('data/projects.json', { cache: 'no-cache' }); + const json = (await result.json()) as ProjectTreeItem[]; + setReport(json); + } catch (e) { + setFetchError(e.message); + } + })(); + }, []); + const [filter, setFilter] = React.useState('Failing'); + + return
+ + +
+
{ + (['Failing', 'All'] as Filter[]).map(item => { + const selected = item === filter; + return
{ + setFilter(item); + }}>{item}
; + }) + }
+ {!fetchError && filter === 'All' && report?.map((project, i) => )} + {!fetchError && filter === 'Failing' && report?.map((project, i) => )} +
+ +
; +}; + +const ProjectTreeItemView: React.FC<{ + project: ProjectTreeItem; + testId?: string, + setTestId: (id: string) => void; + failingOnly?: boolean; +}> = ({ project, testId, setTestId, failingOnly }) => { + return + {statusIconForFailedTests(project.failedTests)}
{project.name || 'Project'}
+ + } loadChildren={() => { + return project.suits.map((s, i) => ) || []; + }} depth={0} expandByDefault={true}>; +}; + +const SuiteTreeItemView: React.FC<{ + suite: SuiteTreeItem, + testId?: string, + setTestId: (id: string) => void; + depth: number, + showFileName: boolean, +}> = ({ suite, testId, setTestId, showFileName, depth }) => { + const location = renderLocation(suite.location, showFileName); + return + {statusIconForFailedTests(suite.failedTests)}
{suite.title}
+ {!!suite.location?.line && location &&
{location}
} + + } loadChildren={() => { + const suiteChildren = suite.suites.map((s, i) => ) || []; + const suiteCount = suite.suites.length; + const testChildren = suite.tests.map((t, i) => ) || []; + return [...suiteChildren, ...testChildren]; + }} depth={depth}>; +}; + +const TestTreeItemView: React.FC<{ + test: TestTreeItem, + showFileName: boolean, + testId?: string, + setTestId: (id: string) => void; + depth: number, +}> = ({ test, testId, setTestId, showFileName, depth }) => { + const fileName = test.location.file; + const name = fileName.substring(fileName.lastIndexOf('/') + 1); + return + {statusIcon(test.outcome)}
{test.title}
+ {showFileName &&
{name}:{test.location.line}
} + {!showFileName &&
{msToString(test.duration)}
} + + } selected={test.testId === testId} depth={depth} onClick={() => setTestId(test.testId)}>; +}; + +const TestCaseView: React.FC<{ + testId: string | undefined, +}> = ({ testId }) => { + const [test, setTest] = React.useState(); + + React.useEffect(() => { + (async () => { + if (!testId) + return; + try { + const result = await fetch(`data/${testId}.json`, { cache: 'no-cache' }); + const json = (await result.json()) as TestCase; + setTest(json); + } catch (e) { + } + })(); + }); + + const [selectedResultIndex, setSelectedResultIndex] = React.useState(0); + return +
+
+
+ { test &&
{test?.title}
} + { test &&
{renderLocation(test.location, true)}
} + { test && ({ + id: String(index), + title:
{statusIcon(result.status)} {retryLabel(index)}
, + render: () => + })) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />} +
+ ; +}; + +const TestResultView: React.FC<{ + test: TestCase, + result: TestResult, +}> = ({ test, result }) => { + return
+ {result.steps.map((step, i) => )} +
; +}; + +const StepTreeItem: React.FC<{ + step: TestStep; + depth: number, +}> = ({ step, depth }) => { + return + {statusIcon(step.error ? 'failed' : 'passed')} + {step.title} +
+
{msToString(step.duration)}
+ } loadChildren={step.steps.length ? () => { + return step.steps.map((s, i) => ); + } : undefined} depth={depth}>; +}; + +function statusIconForFailedTests(failedTests: number) { + return failedTests ? statusIcon('failed') : statusIcon('passed'); +} + +function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expected' | 'unexpected' | 'flaky'): JSX.Element { + switch (status) { + case 'failed': + case 'unexpected': + return ; + case 'passed': + case 'expected': + return ; + case 'timedOut': + return ; + case 'flaky': + return ; + case 'skipped': + return ; + } +} + +function renderLocation(location: Location | undefined, showFileName: boolean) { + if (!location) + return ''; + return (showFileName ? location.file : '') + ':' + location.line; +} + +function retryLabel(index: number) { + if (!index) + return 'Run'; + return `Retry #${index}`; +} diff --git a/src/web/htmlReport2/index.html b/src/web/htmlReport2/index.html new file mode 100644 index 0000000000..f79a89f37d --- /dev/null +++ b/src/web/htmlReport2/index.html @@ -0,0 +1,27 @@ + + + + + + + + Playwright Test Report + + +
+ + diff --git a/src/web/htmlReport2/index.tsx b/src/web/htmlReport2/index.tsx new file mode 100644 index 0000000000..7fbcea2a55 --- /dev/null +++ b/src/web/htmlReport2/index.tsx @@ -0,0 +1,27 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import '../third_party/vscode/codicon.css'; +import * as React from 'react'; +import * as ReactDOM from 'react-dom'; +import { applyTheme } from '../theme'; +import '../common.css'; +import { Report } from './htmlReport'; + +(async () => { + applyTheme(); + ReactDOM.render(, document.querySelector('#root')); +})(); diff --git a/src/web/htmlReport2/webpack.config.js b/src/web/htmlReport2/webpack.config.js new file mode 100644 index 0000000000..279710dfca --- /dev/null +++ b/src/web/htmlReport2/webpack.config.js @@ -0,0 +1,50 @@ +const path = require('path'); +const HtmlWebPackPlugin = require('html-webpack-plugin'); + +const mode = process.env.NODE_ENV === 'production' ? 'production' : 'development'; + +module.exports = { + mode, + entry: { + app: path.join(__dirname, 'index.tsx'), + }, + resolve: { + extensions: ['.ts', '.js', '.tsx', '.jsx'] + }, + devtool: mode === 'production' ? false : 'source-map', + output: { + globalObject: 'self', + filename: '[name].bundle.js', + path: path.resolve(__dirname, '../../../lib/web/htmlReport2') + }, + module: { + rules: [ + { + test: /\.(j|t)sx?$/, + loader: 'babel-loader', + options: { + presets: [ + "@babel/preset-typescript", + "@babel/preset-react" + ] + }, + exclude: /node_modules/ + }, + { + test: /\.css$/, + use: ['style-loader', 'css-loader'] + }, + { + test: /\.ttf$/, + use: ['file-loader'] + } + ] + }, + plugins: [ + new HtmlWebPackPlugin({ + title: 'Playwright Test Report', + template: path.join(__dirname, 'index.html'), + inject: true, + }) + ] +}; diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.ts index c0d67a68d9..a2e42cbb43 100644 --- a/tests/playwright-test/playwright.artifacts.spec.ts +++ b/tests/playwright-test/playwright.artifacts.spec.ts @@ -155,6 +155,8 @@ test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => { 'artifacts-two-contexts-failing', ' test-failed-1.png', ' test-failed-2.png', + 'report', + ' project.report', 'report.json', ]); }); @@ -182,6 +184,8 @@ test('should work with screenshot: only-on-failure', async ({ runInlineTest }, t 'artifacts-two-contexts-failing', ' test-failed-1.png', ' test-failed-2.png', + 'report', + ' project.report', 'report.json', ]); }); @@ -220,6 +224,8 @@ test('should work with trace: on', async ({ runInlineTest }, testInfo) => { 'artifacts-two-contexts-failing', ' trace-1.zip', ' trace.zip', + 'report', + ' project.report', 'report.json', ]); }); @@ -247,6 +253,8 @@ test('should work with trace: retain-on-failure', async ({ runInlineTest }, test 'artifacts-two-contexts-failing', ' trace-1.zip', ' trace.zip', + 'report', + ' project.report', 'report.json', ]); }); @@ -274,6 +282,8 @@ test('should work with trace: on-first-retry', async ({ runInlineTest }, testInf 'artifacts-two-contexts-failing-retry1', ' trace-1.zip', ' trace.zip', + 'report', + ' project.report', 'report.json', ]); }); @@ -314,6 +324,8 @@ test('should stop tracing with trace: on-first-retry, when not retrying', async expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ 'a-shared-flaky-retry1', ' trace.zip', + 'report', + ' project.report', 'report.json', ]); }); diff --git a/tests/playwright-test/raw-reporter.spec.ts b/tests/playwright-test/raw-reporter.spec.ts new file mode 100644 index 0000000000..8119d6cd99 --- /dev/null +++ b/tests/playwright-test/raw-reporter.spec.ts @@ -0,0 +1,119 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import { test, expect } from './playwright-test-fixtures'; + +test('should generate raw report', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('passes', async ({ page }, testInfo) => {}); + `, + }, { usesCustomOutputDir: true }); + const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8')); + expect(json.config).toBeTruthy(); + expect(json.project).toBeTruthy(); + expect(result.exitCode).toBe(0); +}); + +test('should use project name', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [{ + name: 'project-name', + outputDir: 'output' + }] + } + `, + 'a.test.js': ` + const { test } = pwt; + test('passes', async ({ page }, testInfo) => {}); + `, + }, { usesCustomOutputDir: true }); + const json = JSON.parse(fs.readFileSync(testInfo.outputPath('output', 'report', 'project-name.report'), 'utf-8')); + expect(json.project.name).toBe('project-name'); + expect(result.exitCode).toBe(0); +}); + +test('should save stdio', async ({ runInlineTest }, testInfo) => { + await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('passes', async ({ page }, testInfo) => { + console.log('STDOUT'); + console.error('STDERR'); + }); + `, + }, { usesCustomOutputDir: true }); + const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8')); + const result = json.suites[0].tests[0].results[0]; + expect(result.attachments[0].name).toBe('stdout'); + expect(result.attachments[1].name).toBe('stderr'); + const path1 = result.attachments[0].path; + expect(fs.readFileSync(path1, 'utf-8')).toContain('STDOUT'); + const path2 = result.attachments[1].path; + expect(fs.readFileSync(path2, 'utf-8')).toContain('STDERR'); +}); + +test('should save attachments', async ({ runInlineTest }, testInfo) => { + await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('passes', async ({ page }, testInfo) => { + testInfo.attachments.push({ + name: 'binary', + contentType: 'application/octet-stream', + body: Buffer.from([1,2,3]) + }); + testInfo.attachments.push({ + name: 'text', + contentType: 'text/plain', + path: 'dummy-path' + }); + }); + `, + }, { usesCustomOutputDir: true }); + const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8')); + const result = json.suites[0].tests[0].results[0]; + expect(result.attachments[0].name).toBe('binary'); + expect(result.attachments[1].name).toBe('text'); + const path1 = result.attachments[0].path; + expect(fs.readFileSync(path1)).toEqual(Buffer.from([1,2,3])); + const path2 = result.attachments[1].path; + expect(path2).toBe('dummy-path'); +}); + +test('dupe project names', async ({ runInlineTest }, testInfo) => { + await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'project-name' }, + { name: 'project-name' }, + { name: 'project-name' }, + ] + } + `, + 'a.test.js': ` + const { test } = pwt; + test('passes', async ({ page }, testInfo) => {}); + `, + }, { usesCustomOutputDir: true }); + const files = fs.readdirSync(testInfo.outputPath('test-results', 'report')); + expect(new Set(files)).toEqual(new Set(['project-name.report', 'project-name-1.report', 'project-name-2.report'])); +}); diff --git a/utils/build/build.js b/utils/build/build.js index 8587b919df..7e8eb7e3b5 100644 --- a/utils/build/build.js +++ b/utils/build/build.js @@ -115,6 +115,7 @@ const webPackFiles = [ 'src/web/traceViewer/webpack.config.js', 'src/web/recorder/webpack.config.js', 'src/web/htmlReport/webpack.config.js', + 'src/web/htmlReport2/webpack.config.js', ]; for (const file of webPackFiles) { steps.push({ diff --git a/utils/check_deps.js b/utils/check_deps.js index f453a00ff1..cfd2422272 100644 --- a/utils/check_deps.js +++ b/utils/check_deps.js @@ -195,6 +195,7 @@ DEPS['src/test/'] = ['src/test/**', 'src/utils/utils.ts', 'src/utils/**']; // HTML report DEPS['src/web/htmlReport/'] = ['src/test/**', 'src/web/']; +DEPS['src/web/htmlReport2/'] = ['src/test/**', 'src/web/']; checkDeps().catch(e => {