/** * Copyright (c) Microsoft Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import fs from 'fs'; import path from 'path'; import { FullConfig, Location, Suite, TestCase, TestResult, TestStatus, TestStep } from '../../types/testReporter'; import { assert, calculateSha1 } from 'playwright-core/lib/utils/utils'; import { sanitizeForFilePath } from '../util'; import { formatResultFailure } from './base'; import { toPosixPath, serializePatterns } from './json'; import { MultiMap } from 'playwright-core/lib/utils/multimap'; import { codeFrameColumns } from '@babel/code-frame'; export type JsonLocation = Location; export type JsonError = string; export type JsonStackFrame = { file: string, line: number, column: number }; export type JsonReport = { config: JsonConfig, project: JsonProject, suites: JsonSuite[], }; export type JsonConfig = Omit; export type JsonProject = { metadata: any, name: string, outputDir: string, repeatEach: number, retries: number, testDir: string, testIgnore: string[], testMatch: string[], timeout: number, }; export type JsonSuite = { fileId: string; title: string; location?: JsonLocation; suites: JsonSuite[]; tests: JsonTestCase[]; }; export type JsonTestCase = { testId: string; title: string; location: JsonLocation; expectedStatus: TestStatus; timeout: number; annotations: { type: string, description?: string }[]; retries: number; results: JsonTestResult[]; ok: boolean; outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; }; export type JsonAttachment = { name: string; body?: string | Buffer; path?: string; contentType: string; }; export type JsonTestResult = { retry: number; workerIndex: number; startTime: string; duration: number; status: TestStatus; errors: JsonError[]; attachments: JsonAttachment[]; steps: JsonTestStep[]; }; export type JsonTestStep = { title: string; category: string, startTime: string; duration: number; error?: JsonError; steps: JsonTestStep[]; location?: Location; snippet?: string; count: number; }; class RawReporter { private config!: FullConfig; private suite!: Suite; private stepsInFile = new MultiMap(); onBegin(config: FullConfig, suite: Suite) { this.config = config; this.suite = suite; } async onEnd() { const projectSuites = this.suite.suites; for (const suite of projectSuites) { const project = suite.project(); assert(project, 'Internal Error: Invalid project structure'); const reportFolder = path.join(project.outputDir, 'report'); fs.mkdirSync(reportFolder, { recursive: true }); let reportFile: string | undefined; for (let i = 0; i < 10; ++i) { reportFile = path.join(reportFolder, sanitizeForFilePath(project.name || 'project') + (i ? '-' + i : '') + '.report'); try { if (fs.existsSync(reportFile)) continue; } catch (e) { } break; } if (!reportFile) throw new Error('Internal error, could not create report file'); const report = this.generateProjectReport(this.config, suite); fs.writeFileSync(reportFile, JSON.stringify(report, undefined, 2)); } } generateAttachments(config: FullConfig): JsonAttachment[] { return this._createAttachments(config.attachments); } generateProjectReport(config: FullConfig, suite: Suite): JsonReport { this.config = config; const project = suite.project(); assert(project, 'Internal Error: Invalid project structure'); const report: JsonReport = { config, project: { metadata: project.metadata, name: project.name, outputDir: project.outputDir, repeatEach: project.repeatEach, retries: project.retries, testDir: project.testDir, testIgnore: serializePatterns(project.testIgnore), testMatch: serializePatterns(project.testMatch), timeout: project.timeout, }, suites: suite.suites.map(fileSuite => { // fileId is based on the location of the enclosing file suite. // Don't use the file in test/suite location, it can be different // due to the source map / require. const fileId = calculateSha1(fileSuite.location!.file.split(path.sep).join('/')); return this._serializeSuite(fileSuite, fileId); }) }; for (const file of this.stepsInFile.keys()) { let source: string; try { source = fs.readFileSync(file, 'utf-8') + '\n//'; } catch (e) { continue; } const lines = source.split('\n').length; const highlighted = codeFrameColumns(source, { start: { line: lines, column: 1 } }, { highlightCode: true, linesAbove: lines, linesBelow: 0 }); const highlightedLines = highlighted.split('\n'); const lineWithArrow = highlightedLines[highlightedLines.length - 1]; for (const step of this.stepsInFile.get(file)) { // Don't bother with snippets that have less than 3 lines. if (step.location!.line < 2 || step.location!.line >= lines) continue; // Cut out snippet. const snippetLines = highlightedLines.slice(step.location!.line - 2, step.location!.line + 1); // Relocate arrow. const index = lineWithArrow.indexOf('^'); const shiftedArrow = lineWithArrow.slice(0, index) + ' '.repeat(step.location!.column - 1) + lineWithArrow.slice(index); // Insert arrow line. snippetLines.splice(2, 0, shiftedArrow); step.snippet = snippetLines.join('\n'); } } return report; } private _serializeSuite(suite: Suite, fileId: string): JsonSuite { const location = this._relativeLocation(suite.location); return { title: suite.title, fileId, location, suites: suite.suites.map(s => this._serializeSuite(s, fileId)), tests: suite.tests.map(t => this._serializeTest(t, fileId)), }; } private _serializeTest(test: TestCase, fileId: string): JsonTestCase { const [, projectName, , ...titles] = test.titlePath(); const testIdExpression = `project:${projectName}|path:${titles.join('>')}|repeat:${test.repeatEachIndex}`; const testId = fileId + '-' + calculateSha1(testIdExpression); return { testId, title: test.title, location: this._relativeLocation(test.location)!, expectedStatus: test.expectedStatus, timeout: test.timeout, annotations: test.annotations, retries: test.retries, ok: test.ok(), outcome: test.outcome(), results: test.results.map(r => this._serializeResult(test, r)), }; } private _serializeResult(test: TestCase, result: TestResult): JsonTestResult { return { retry: result.retry, workerIndex: result.workerIndex, startTime: result.startTime.toISOString(), duration: result.duration, status: result.status, errors: formatResultFailure(this.config, test, result, '', true).map(error => error.message), attachments: this._createAttachments(result.attachments, result), steps: dedupeSteps(result.steps.map(step => this._serializeStep(test, step))) }; } private _serializeStep(test: TestCase, step: TestStep): JsonTestStep { const result: JsonTestStep = { title: step.title, category: step.category, startTime: step.startTime.toISOString(), duration: step.duration, error: step.error?.message, location: this._relativeLocation(step.location), steps: dedupeSteps(step.steps.map(step => this._serializeStep(test, step))), count: 1 }; if (step.location) this.stepsInFile.set(step.location.file, result); return result; } private _createAttachments(attachments: TestResult['attachments'], ioStreams?: Pick): JsonAttachment[] { const out: JsonAttachment[] = []; for (const attachment of attachments) { if (attachment.body) { out.push({ name: attachment.name, contentType: attachment.contentType, body: attachment.body }); } else if (attachment.path) { out.push({ name: attachment.name, contentType: attachment.contentType, path: attachment.path }); } } if (ioStreams) { for (const chunk of ioStreams.stdout) out.push(this._stdioAttachment(chunk, 'stdout')); for (const chunk of ioStreams.stderr) out.push(this._stdioAttachment(chunk, 'stderr')); } return out; } private _stdioAttachment(chunk: Buffer | string, type: 'stdout' | 'stderr'): JsonAttachment { if (typeof chunk === 'string') { return { name: type, contentType: 'text/plain', body: chunk }; } return { name: type, contentType: 'application/octet-stream', body: chunk }; } private _relativeLocation(location: Location | undefined): Location | undefined { if (!location) return undefined; const file = toPosixPath(path.relative(this.config.rootDir, location.file)); return { file, line: location.line, column: location.column, }; } } function dedupeSteps(steps: JsonTestStep[]): JsonTestStep[] { const result: JsonTestStep[] = []; let lastStep: JsonTestStep | undefined; for (const step of steps) { const canDedupe = !step.error && step.duration >= 0 && step.location?.file && !step.steps.length; if (canDedupe && lastStep && step.category === lastStep.category && step.title === lastStep.title && step.location?.file === lastStep.location?.file && step.location?.line === lastStep.location?.line && step.location?.column === lastStep.location?.column) { ++lastStep.count; lastStep.duration += step.duration; continue; } result.push(step); lastStep = canDedupe ? step : undefined; } return result; } export default RawReporter;