From 297fea082604450d99e526e9654092d42f3b533b Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 1 May 2023 09:15:08 -0700 Subject: [PATCH] chore: purify the junit reporter (#22624) --- docs/src/test-reporters-js.md | 74 ------------- .../playwright-test/src/reporters/junit.ts | 100 ++++-------------- tests/playwright-test/reporter-junit.spec.ts | 92 ++++------------ 3 files changed, 41 insertions(+), 225 deletions(-) diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 79d02556f3..7a4742fa5d 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -268,80 +268,6 @@ export default defineConfig({ }); ``` -The JUnit reporter provides support for embedding additional information on the `testcase` elements using inner `properties`. This is based on an [evolved JUnit XML format](https://docs.getxray.app/display/XRAYCLOUD/Taking+advantage+of+JUnit+XML+reports) from Xray Test Management, but can also be used by other tools if they support this way of embedding additional information for test results; please check it first. - -In configuration file, a set of options can be used to configure this behavior. A full example, in this case for Xray, follows ahead. - -```js -import { defineConfig } from '@playwright/test'; - -// JUnit reporter config for Xray -const xrayOptions = { - // Whether to add with all annotations; default is false - embedAnnotationsAsProperties: true, - - // By default, annotation is reported as . - // These annotations are reported as value. - textContentAnnotations: ['test_description'], - - // This will create a "testrun_evidence" property that contains all attachments. Each attachment is added as an inner element. - // Disables [[ATTACHMENT|path]] in the . - embedAttachmentsAsProperty: 'testrun_evidence', - - // Where to put the report. - outputFile: './xray-report.xml' -}; - -export default defineConfig({ - reporter: [['junit', xrayOptions]] -}); -``` - -In the previous configuration sample, all annotations will be added as `` elements on the JUnit XML report. The annotation type is mapped to the `name` attribute of the ``, and the annotation description will be added as a `value` attribute. In this case, the exception will be the annotation type `testrun_evidence` whose description will be added as inner content on the respective ``. -Annotations can be used to, for example, link a Playwright test with an existing Test in Xray or to link a test with an existing story/requirement in Jira (i.e., "cover" it). - -```js -// example.spec.ts/js -import { test } from '@playwright/test'; - -test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => { - testInfo.annotations.push({ type: 'test_id', description: '1234' }); - testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' }); - testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' }); - testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' }); - testInfo.annotations.push({ type: 'test_description', description: 'sample description' }); -}); -``` - -Please note that the semantics of these properties will depend on the tool that will process this evolved report format; there are no standard property names/annotations. - -If the configuration option `embedAttachmentsAsProperty` is defined, then a `property` with its name is created. Attachments, including their contents, will be embedded on the JUnit XML report inside `` elements under this `property`. Attachments are obtained from the `TestInfo` object, using either a path or a body, and are added as base64 encoded content. -Embedding attachments can be used to attach screenshots or any other relevant evidence; nevertheless, use it wisely as it affects the report size. - -The following configuration sample enables embedding attachments by using the `testrun_evidence` element on the JUnit XML report: - -```js -import { defineConfig } from '@playwright/test'; - -export default defineConfig({ - reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]], -}); -``` - -The following test adds attachments: - -```js -// example.spec.ts/js -import { test } from '@playwright/test'; - -test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => { - const file = testInfo.outputPath('evidence1.txt'); - require('fs').writeFileSync(file, 'hello', 'utf8'); - await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' }); - await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' }); -}); -``` - ### GitHub Actions annotations You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions. diff --git a/packages/playwright-test/src/reporters/junit.ts b/packages/playwright-test/src/reporters/junit.ts index 1e5cf6df4a..fa457b4b05 100644 --- a/packages/playwright-test/src/reporters/junit.ts +++ b/packages/playwright-test/src/reporters/junit.ts @@ -31,17 +31,10 @@ class JUnitReporter implements Reporter { private totalSkipped = 0; private outputFile: string | undefined; private stripANSIControlSequences = false; - private embedAnnotationsAsProperties = false; - private textContentAnnotations: string[] | undefined; - private embedAttachmentsAsProperty: string | undefined; - - constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean, embedAnnotationsAsProperties?: boolean, textContentAnnotations?: string[], embedAttachmentsAsProperty?: string } = {}) { + constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean } = {}) { this.outputFile = options.outputFile || reportOutputNameFromEnv(); this.stripANSIControlSequences = options.stripANSIControlSequences || false; - this.embedAnnotationsAsProperties = options.embedAnnotationsAsProperties || false; - this.textContentAnnotations = options.textContentAnnotations || []; - this.embedAttachmentsAsProperty = options.embedAttachmentsAsProperty; } printsToStdio() { @@ -153,71 +146,15 @@ class JUnitReporter implements Reporter { children: [] as XMLEntry[] }; - if (this.embedAnnotationsAsProperties && test.annotations) { - for (const annotation of test.annotations) { - if (this.textContentAnnotations?.includes(annotation.type)) { - const property: XMLEntry = { - name: 'property', - attributes: { - name: annotation.type - }, - text: annotation.description - }; - properties.children?.push(property); - } else { - const property: XMLEntry = { - name: 'property', - attributes: { - name: annotation.type, - value: (annotation?.description ? annotation.description : '') - } - }; - properties.children?.push(property); - } - } - } - - const systemErr: string[] = []; - // attachments are optionally embed as base64 encoded content on inner elements - if (this.embedAttachmentsAsProperty) { - const evidence: XMLEntry = { + for (const annotation of test.annotations) { + const property: XMLEntry = { name: 'property', attributes: { - name: this.embedAttachmentsAsProperty - }, - children: [] as XMLEntry[] - }; - for (const result of test.results) { - for (const attachment of result.attachments) { - let contents; - if (attachment.body) { - contents = attachment.body.toString('base64'); - } else { - if (!attachment.path) - continue; - try { - if (fs.existsSync(attachment.path)) - contents = fs.readFileSync(attachment.path, { encoding: 'base64' }); - else - systemErr.push(`\nWarning: attachment ${attachment.path} is missing`); - } catch (e) { - } - } - - if (contents) { - const item: XMLEntry = { - name: 'item', - attributes: { - name: attachment.name - }, - text: contents - }; - evidence.children?.push(item); - } - + name: annotation.type, + value: (annotation?.description ? annotation.description : '') } - } - properties.children?.push(evidence); + }; + properties.children?.push(property); } if (properties.children?.length) @@ -240,21 +177,20 @@ class JUnitReporter implements Reporter { } const systemOut: string[] = []; + const systemErr: string[] = []; for (const result of test.results) { systemOut.push(...result.stdout.map(item => item.toString())); systemErr.push(...result.stderr.map(item => item.toString())); - if (!this.embedAttachmentsAsProperty) { - for (const attachment of result.attachments) { - if (!attachment.path) - continue; - try { - const attachmentPath = path.relative(this.config.rootDir, attachment.path); - if (fs.existsSync(attachment.path)) - systemOut.push(`\n[[ATTACHMENT|${attachmentPath}]]\n`); - else - systemErr.push(`\nWarning: attachment ${attachmentPath} is missing`); - } catch (e) { - } + for (const attachment of result.attachments) { + if (!attachment.path) + continue; + try { + const attachmentPath = path.relative(this.config.rootDir, attachment.path); + if (fs.existsSync(attachment.path)) + systemOut.push(`\n[[ATTACHMENT|${attachmentPath}]]\n`); + else + systemErr.push(`\nWarning: attachment ${attachmentPath} is missing`); + } catch (e) { } } } diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts index b7e227bf8b..c4bbbecd1b 100644 --- a/tests/playwright-test/reporter-junit.spec.ts +++ b/tests/playwright-test/reporter-junit.spec.ts @@ -308,48 +308,41 @@ function parseXML(xml: string): any { return result; } -test('should not render annotations to custom testcase properties by default', async ({ runInlineTest }) => { +test('should render annotations to custom testcase properties', async ({ runInlineTest }) => { const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - testInfo.annotations.push({ type: 'unknown_annotation', description: 'unknown' }); - });2 - ` - }, { reporter: 'junit' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['properties']).not.toBeTruthy(); - expect(result.exitCode).toBe(0); -}); - -test('should render text content based annotations to custom testcase properties', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - const xrayOptions = { - embedAnnotationsAsProperties: true, - textContentAnnotations: ['test_description'] - } - module.exports = { - reporter: [ ['junit', xrayOptions] ], - }; - `, + 'playwright.config.ts': `module.exports = { reporter: 'junit' };`, 'a.test.js': ` import { test, expect } from '@playwright/test'; test('one', async ({}, testInfo) => { testInfo.annotations.push({ type: 'test_description', description: 'sample description' }); - testInfo.annotations.push({ type: 'unknown_annotation', description: 'unknown' }); }); ` }, { reporter: '' }); const xml = parseXML(result.output); const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; expect(testcase['properties']).toBeTruthy(); - expect(testcase['properties'][0]['property'].length).toBe(2); + expect(testcase['properties'][0]['property'].length).toBe(1); expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description'); - expect(testcase['properties'][0]['property'][0]['_']).toBe('\nsample description\n'); - expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('unknown_annotation'); - expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('unknown'); + expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('sample description'); + expect(result.exitCode).toBe(0); +}); + +test('should render built-in annotations to testcase properties', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': `module.exports = { reporter: 'junit' };`, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + test.slow(); + }); + ` + }, { reporter: '' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).toBeTruthy(); + expect(testcase['properties'][0]['property'].length).toBe(1); + expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('slow'); + expect(testcase['properties'][0]['property'][0]['$']['value']).toBe(''); expect(result.exitCode).toBe(0); }); @@ -388,45 +381,6 @@ test('should render all annotations to testcase value based properties, if reque expect(result.exitCode).toBe(0); }); -test('should embed attachments to a custom testcase property, if explicitly requested', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - const xrayOptions = { - embedAttachmentsAsProperty: 'testrun_evidence' - } - module.exports = { - reporter: [ ['junit', xrayOptions] ], - }; - `, - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - const file = testInfo.outputPath('evidence1.txt'); - require('fs').writeFileSync(file, 'hello', 'utf8'); - testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' }); - testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' }); - // await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' }); - // await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' }); - console.log('log here'); - }); - ` - }, { reporter: '' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['properties']).toBeTruthy(); - expect(testcase['properties'][0]['property'].length).toBe(1); - expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('testrun_evidence'); - expect(testcase['properties'][0]['property'][0]['item'][0]['$']['name']).toBe('evidence1.txt'); - expect(testcase['properties'][0]['property'][0]['item'][0]['_']).toBe('\naGVsbG8=\n'); - expect(testcase['properties'][0]['property'][0]['item'][1]['$']['name']).toBe('evidence2.txt'); - expect(testcase['properties'][0]['property'][0]['item'][1]['_']).toBe('\nd29ybGQ=\n'); - expect(testcase['system-out'].length).toBe(1); - expect(testcase['system-out'][0].trim()).toBe([ - `log here` - ].join('\n')); - expect(result.exitCode).toBe(0); -}); - test('should not embed attachments to a custom testcase property, if not explicitly requested', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.js': `