Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix(merge): include command hash into the report name

Browse source 
Reference https://github.com/microsoft/playwright/issues/30091
This commit is contained in:
Yury Semikhatsky 2024-04-24 15:56:05 -07:00
parent 59689c9c97
commit 61d2e2b96b
5 changed files with 40 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
packages/playwright/src/common/config.ts
View file

@ -48,6 +48,7 @@ export class FullConfigInternal {
readonly webServers: NonNullable<FullConfig['webServer']>[];
readonly plugins: TestRunnerPluginRegistration[];
readonly projects: FullProjectInternal[] = [];
commandHash: string = '';
cliArgs: string[] = [];
cliGrep: string | undefined;
cliGrepInvert: string | undefined;

16
packages/playwright/src/program.ts
View file

@ -20,7 +20,7 @@ import type { Command } from 'playwright-core/lib/utilsBundle';
import fs from 'fs';
import path from 'path';
import { Runner } from './runner/runner';
import { stopProfiling, startProfiling, gracefullyProcessExitDoNotHang } from 'playwright-core/lib/utils';
import { stopProfiling, startProfiling, gracefullyProcessExitDoNotHang, calculateSha1 } from 'playwright-core/lib/utils';
import { serializeError } from './util';
import { showHTMLReport } from './reporters/html';
import { createMergedReport } from './reporters/merge';
@ -183,6 +183,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
if (!config)
return;
config.commandHash = computeCommandHash(args, opts);
config.cliArgs = args;
config.cliGrep = opts.grep as string | undefined;
config.cliGrepInvert = opts.grepInvert as string | undefined;
@ -201,6 +202,19 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
gracefullyProcessExitDoNotHang(exitCode);
}
function computeCommandHash(args: string[], opts: { [key: string]: any }) {
const command = {
args,
opts: {} as any
};
// Omit shard from the hash, shard number is added explicitely at the end of the blob report file name.
const keys = Object.keys(opts).filter(k => k !== 'shard');
if (!keys.length && !args.length)
return '';
keys.sort().forEach(key => command.opts[key] = opts[key]);
return calculateSha1(JSON.stringify(command)).substring(0, 7);
}
async function runTestServer(opts: { [key: string]: any }) {
const host = opts.host || 'localhost';
const port = opts.port ? +opts.port : 0;

3
packages/playwright/src/reporters/blob.ts
View file

@ -30,6 +30,7 @@ type BlobReporterOptions = {
configDir: string;
outputDir?: string;
fileName?: string;
_commandHash: string;
};
export const currentBlobReportVersion = 2;
@ -112,6 +113,8 @@ export class BlobReporter extends TeleReporterEmitter {
if (this._options.fileName)
return this._options.fileName;
let reportName = 'report';
if (this._options._commandHash)
reportName += '-' + this._options._commandHash;
if (config.shard) {
const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0');
reportName = `${reportName}-${paddedNumber}`;

1
packages/playwright/src/runner/reporters.ts
View file

@ -90,6 +90,7 @@ function reporterOptions(config: FullConfigInternal, mode: 'list' | 'test' | 'me
configDir: config.configDir,
_mode: mode,
_isTestServer: isTestServer,
_commandHash: config.commandHash,
};
}

40
tests/playwright-test/reporter-blob.spec.ts
View file

@ -274,7 +274,7 @@ test('should merge blob into blob', async ({ runInlineTest, mergeReports, showRe
{
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
expect(reportFiles).toEqual(['report-873189c-1.zip', 'report-873189c-2.zip']);
const { exitCode } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'blob'] });
expect(exitCode).toBe(0);
}
@ -440,7 +440,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip', 'report-3.zip']);
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/, /report-.*-3.zip/].map(r => expect.stringMatching(r)));
const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] });
expect(exitCode).toBe(0);
@ -519,7 +519,7 @@ test('should print progress', async ({ runInlineTest, mergeReports }) => {
await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r)));
const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0);
@ -570,7 +570,7 @@ test('preserve attachments', async ({ runInlineTest, mergeReports, showReport, p
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip']);
expect(reportFiles).toEqual(['report-873189c-1.zip']);
const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0);
@ -633,7 +633,7 @@ test('generate html with attachment urls', async ({ runInlineTest, mergeReports,
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip']);
expect(reportFiles).toEqual(['report-873189c-1.zip']);
const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0);
@ -707,7 +707,7 @@ test('resource names should not clash between runs', async ({ runInlineTest, sho
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r)));
const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0);
@ -780,7 +780,7 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip']);
expect(reportFiles).toEqual([expect.stringMatching(/report-.*-1.zip/)]);
const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html,line'] });
expect(exitCode).toBe(0);
@ -841,7 +841,7 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r)));
const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] });
expect(exitCode).toBe(0);
@ -988,7 +988,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report-1.zip', 'report-3.zip']);
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-3.zip/].map(r => expect.stringMatching(r)));
const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js'), '-c', test.info().outputPath('merge.config.ts')] });
expect(exitCode).toBe(0);
const json = JSON.parse(fs.readFileSync(test.info().outputPath('config.json')).toString());
@ -1144,7 +1144,7 @@ test('preserve steps in html report', async ({ runInlineTest, mergeReports, show
await runInlineTest(files);
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual(['report.zip']);
expect(reportFiles).toEqual(['report-873189c.zip']);
// Run merger in a different directory to make sure relative paths will not be resolved
// relative to the current directory.
const mergeCwd = test.info().outputPath('foo');
@ -1277,9 +1277,9 @@ test('blob report should include version', async ({ runInlineTest }) => {
// CI/1 is a part of user agent string, make sure it matches in the nested test runner.
await runInlineTest(files, undefined, { CI: process.env.CI });
const reportFiles = await fs.promises.readdir(reportDir);
expect(reportFiles).toEqual(['report.zip']);
expect(reportFiles).toEqual(['report-873189c.zip']);
const events = await extractReport(test.info().outputPath('blob-report', 'report.zip'), test.info().outputPath('tmp'));
const events = await extractReport(test.info().outputPath('blob-report', 'report-873189c.zip'), test.info().outputPath('tmp'));
const metadataEvent = events.find(e => e.method === 'onBlobReportMetadata');
expect(metadataEvent.params.version).toBe(2);
expect(metadataEvent.params.userAgent).toBe(getUserAgent());
@ -1330,7 +1330,7 @@ test('merge-reports should throw if report version is from the future', async ({
await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
const reportFiles = await fs.promises.readdir(reportDir);
expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r)));
// Extract report and modify version.
const reportZipFile = test.info().outputPath('blob-report', reportFiles[1]);
@ -1346,7 +1346,7 @@ test('merge-reports should throw if report version is from the future', async ({
const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(1);
expect(output).toContain(`Error: Blob report report-2.zip was created with a newer version of Playwright.`);
expect(output).toContain(`Error: Blob report report-873189c-2.zip was created with a newer version of Playwright.`);
});
@ -1383,7 +1383,7 @@ test('should merge blob reports with same name', async ({ runInlineTest, mergeRe
`
};
await runInlineTest(files);
const reportZip = test.info().outputPath('blob-report', 'report.zip');
const reportZip = test.info().outputPath('blob-report', 'report-873189c.zip');
const allReportsDir = test.info().outputPath('all-blob-reports');
await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-1.zip'));
await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-2.zip'));
@ -1485,8 +1485,8 @@ test('merge reports with different rootDirs', async ({ runInlineTest, mergeRepor
await runInlineTest(files2, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir2/playwright.config.ts')] });
const allReportsDir = test.info().outputPath('all-blob-reports');
await fs.promises.cp(test.info().outputPath('dir1', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-1.zip'));
await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-2.zip'));
await fs.promises.cp(test.info().outputPath('dir1', 'blob-report', 'report-546b561.zip'), path.join(allReportsDir, 'report-1.zip'));
await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report-f033fe4.zip'), path.join(allReportsDir, 'report-2.zip'));
{
const { exitCode, output } = await mergeReports(allReportsDir);
@ -1601,7 +1601,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn
await fs.promises.mkdir(allReportsDir, { recursive: true });
// Extract report and change path separators.
const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report.zip');
const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report-e4e2def.zip');
const events = await extractReport(reportZipFile, test.info().outputPath('tmp'));
events.forEach(patchPathSeparators);
@ -1610,7 +1610,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn
await zipReport(events, report1);
// Copy second report as is.
await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-2.zip'));
await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report-b754021.zip'), path.join(allReportsDir, 'report-2.zip'));
{
const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] });
@ -1655,7 +1655,7 @@ test('merge reports without --config preserves path separators', async ({ runInl
await fs.promises.mkdir(allReportsDir, { recursive: true });
// Extract report and change path separators.
const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report.zip');
const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report-6fabe8b.zip');
const events = await extractReport(reportZipFile, test.info().outputPath('tmp'));
events.forEach(patchPathSeparators);