diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 015dbc1e71..0caa6e32b5 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -265,7 +265,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] | return reporters; } -export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'markdown'] as const; +export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'markdown', 'lastrun'] as const; export type BuiltInReporter = typeof builtInReporters[number]; export type ContextReuseMode = 'none' | 'when-possible'; diff --git a/packages/playwright/src/reporters/lastrun.ts b/packages/playwright/src/reporters/lastrun.ts new file mode 100644 index 0000000000..c23cce0776 --- /dev/null +++ b/packages/playwright/src/reporters/lastrun.ts @@ -0,0 +1,80 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import fs from 'fs'; +import path from 'path'; +import type { FullResult, TestCase, TestResult } from '../../types/testReporter'; +import type { LastRunInfo } from '../runner/runner'; +import { BaseReporter, resolveOutputFile } from './base'; + +type LastRunOptions = { + outputFile?: string, + configDir: string, + _mode: 'list' | 'test' | 'merge', +}; + +// The blob reporter adds a suffix to test ids to avoid collisions. But we need +// to remove that suffix to match the test ids from the last run. +function unsaltTestId(testId: string): string { + return testId.split('-').map(s => s.substring(0, 20)).join('-'); +} + +class LastRunReporter extends BaseReporter { + + private lastRun: LastRunInfo = { + failedTests: [], + status: 'passed', + testDurations: {}, + }; + + private resolvedOutputFile: string | undefined; + private testId: (testId: string) => string; + + constructor(options: LastRunOptions) { + super(); + this.resolvedOutputFile = resolveOutputFile('LASTRUN', { fileName: '.last-run.json', ...options })?.outputFile; + this.testId = options._mode === 'merge' ? unsaltTestId : (testId: string) => testId; + } + + override printsToStdio() { + return !this.resolvedOutputFile; + } + + override onTestEnd(test: TestCase, result: TestResult): void { + super.onTestEnd(test, result); + this.lastRun.testDurations![this.testId(test.id)] = result.duration; + if (result.status === 'failed') + this.lastRun.failedTests.push(this.testId(test.id)); + } + + override async onEnd(result: FullResult) { + await super.onEnd(result); + this.lastRun.status = result.status; + await this.outputReport(this.lastRun, this.resolvedOutputFile); + } + + async outputReport(lastRun: LastRunInfo, resolvedOutputFile: string | undefined) { + const reportString = JSON.stringify(lastRun, undefined, 2); + if (resolvedOutputFile) { + await fs.promises.mkdir(path.dirname(resolvedOutputFile), { recursive: true }); + await fs.promises.writeFile(resolvedOutputFile, reportString); + } else { + console.log(reportString); + } + } +} + +export default LastRunReporter; + diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index cbad53e892..227ebd568b 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -23,6 +23,7 @@ import GitHubReporter from '../reporters/github'; import HtmlReporter from '../reporters/html'; import JSONReporter from '../reporters/json'; import JUnitReporter from '../reporters/junit'; +import LastRunReporter from '../reporters/lastrun'; import LineReporter from '../reporters/line'; import ListReporter from '../reporters/list'; import MarkdownReporter from '../reporters/markdown'; @@ -46,6 +47,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | null: EmptyReporter, html: HtmlReporter, markdown: MarkdownReporter, + lastrun: LastRunReporter, }; const reporters: ReporterV2[] = []; descriptions ??= config.config.reporter; diff --git a/tests/playwright-test/reporter-lastrun.spec.ts b/tests/playwright-test/reporter-lastrun.spec.ts new file mode 100644 index 0000000000..4560684468 --- /dev/null +++ b/tests/playwright-test/reporter-lastrun.spec.ts @@ -0,0 +1,195 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as fs from 'fs'; +import { expect, test } from './playwright-test-fixtures'; + +type LastRunReport = { + status: string; + failedTests: string[]; + testDurations: { [testId: string]: number }; +}; + +const testCases = { + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('a', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('math fails!', async ({}) => { + expect(1 + 1).toBe(3); + }); + test.skip('math skipped', async ({}) => { + }); + ` +}; + +test('report lastrun info', async ({ runInlineTest }) => { + const result = await runInlineTest(testCases, { reporter: 'lastrun' }); + expect(result.exitCode).toBe(1); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const lastRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + expect(lastRun.status).toEqual('failed'); + expect(lastRun.failedTests.length).toEqual(1); + expect(Object.keys(lastRun.testDurations).length).toEqual(3); +}); + +test('keep test-ids consistent when re-run', async ({ runInlineTest }) => { + let lastRun: LastRunReport; + { + const result = await runInlineTest(testCases, { reporter: 'lastrun' }); + expect(result.exitCode).toBe(1); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const currentRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + expect(currentRun.failedTests.length).toBeGreaterThanOrEqual(1); + expect(Object.keys(currentRun.testDurations).length).toBeGreaterThanOrEqual(1); + lastRun = currentRun; + } + { + const result = await runInlineTest(testCases, { reporter: 'lastrun' }); + expect(result.exitCode).toBe(1); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const currentRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + // Ensure test-ids are the same as the previous run. + expect(currentRun.failedTests.sort()).toEqual(lastRun.failedTests.sort()); + expect(Object.keys(currentRun.testDurations).sort()).toEqual(Object.keys(lastRun.testDurations).sort()); + } +}); + +test('keep test-ids consistent when merging reports', async ({ runInlineTest, mergeReports }) => { + const reportDir = test.info().outputPath('blob-report'); + const allFailedTests: string[] = []; + const allTestDurations: { [testId: string]: number } = {}; + const testFiles = { + ...testCases, + 'playwright.config.ts': `module.exports = { + reporter: [ + ['lastrun'], + ['blob', { outputDir: 'blob-report' }], + ] + };`, + }; + { + await runInlineTest(testFiles, { shard: '1/2' }); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const lastRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + lastRun.failedTests.forEach(t => allFailedTests.push(t)); + Object.entries(lastRun.testDurations).forEach(([k, v]) => allTestDurations[k] = v); + } + { + await runInlineTest(testFiles, { shard: '2/2' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const lastRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + lastRun.failedTests.forEach(t => allFailedTests.push(t)); + Object.entries(lastRun.testDurations).forEach(([k, v]) => allTestDurations[k] = v); + } + { + expect(allFailedTests.length).toBeGreaterThanOrEqual(1); + expect(Object.keys(allTestDurations).length).toBeGreaterThanOrEqual(1); + } + { + const reportFiles = await fs.promises.readdir(reportDir); + reportFiles.sort(); + expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); + const result = await mergeReports(reportDir, { 'PLAYWRIGHT_HTML_OPEN': 'never' }, { additionalArgs: ['--reporter', 'lastrun'] }); + expect(result.exitCode).toBe(0); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const lastRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + // Ensure test-ids are the same as the previous run. + expect(Object.keys(lastRun.testDurations).sort()).toEqual(Object.keys(allTestDurations).sort()); + expect(lastRun.failedTests.sort()).toEqual(allFailedTests.sort()); + } +}); + +test('keep existing test-ids when test files are modified', async ({ runInlineTest }) => { + let firstTestId: string; + { + // First we start with a single test and record the test-id. + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('first test created', async ({}) => { + expect(1 + 1).toBe(2); + }); + ` + }, { reporter: 'lastrun' }); + expect(result.exitCode).toBe(0); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const lastRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + expect(lastRun.failedTests.length).toEqual(0); + expect(Object.keys(lastRun.testDurations).length).toEqual(1); + firstTestId = Object.keys(lastRun.testDurations)[0]; + } + { + // Then we add more tests and ensure the test-id is still the same. + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('another test', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('first test created', async ({}) => { + expect(1 + 1).toBe(2); + }); + test('yet another test', async ({}) => { + expect(1 + 1).toBe(2); + }); + ` + }, { reporter: 'lastrun' }); + expect(result.exitCode).toBe(0); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const lastRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + expect(lastRun.failedTests.length).toEqual(0); + expect(Object.keys(lastRun.testDurations).length).toEqual(3); + expect(Object.keys(lastRun.testDurations)).toContain(firstTestId); + } +}); + +test('ensure same tests in different files have distinct test-ids', async ({ runInlineTest }) => { + let firstTestId: string; + { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('math test', async ({}) => { + expect(1 + 1).toBe(2); + }); + ` + }, { reporter: 'lastrun' }, {}, { additionalArgs: ['a.test.js'] }); + expect(result.exitCode).toBe(0); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const lastRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + expect(lastRun.failedTests.length).toEqual(0); + expect(Object.keys(lastRun.testDurations).length).toEqual(1); + firstTestId = Object.keys(lastRun.testDurations)[0]; + } + { + const result = await runInlineTest({ + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('math test', async ({}) => { + expect(1 + 1).toBe(2); + }); + ` + }, { reporter: 'lastrun' }, {}, { additionalArgs: ['b.test.js'] }); + expect(result.exitCode).toBe(0); + const lastRunFilename = test.info().outputPath('.last-run.json'); + const lastRun = JSON.parse(await fs.promises.readFile(lastRunFilename, 'utf8')) as LastRunReport; + expect(lastRun.failedTests.length).toEqual(0); + expect(Object.keys(lastRun.testDurations).length).toEqual(1); + const otherTestId = Object.keys(lastRun.testDurations)[0]; + expect(otherTestId).not.toEqual(firstTestId); + } +});