Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

project name in report file

Browse source
This commit is contained in:
Yury Semikhatsky 2024-04-24 17:58:19 -07:00
parent 61d2e2b96b
commit b5a6ce226d
5 changed files with 74 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
packages/playwright/src/common/config.ts
View file

@ -48,7 +48,6 @@ export class FullConfigInternal {
readonly webServers: NonNullable<FullConfig['webServer']>[]; readonly webServers: NonNullable<FullConfig['webServer']>[];
readonly plugins: TestRunnerPluginRegistration[]; readonly plugins: TestRunnerPluginRegistration[];
readonly projects: FullProjectInternal[] = []; readonly projects: FullProjectInternal[] = [];
commandHash: string = '';
cliArgs: string[] = []; cliArgs: string[] = [];
cliGrep: string | undefined; cliGrep: string | undefined;
cliGrepInvert: string | undefined; cliGrepInvert: string | undefined;

16
packages/playwright/src/program.ts
View file

@ -20,7 +20,7 @@ import type { Command } from 'playwright-core/lib/utilsBundle';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { Runner } from './runner/runner'; import { Runner } from './runner/runner';
import { stopProfiling, startProfiling, gracefullyProcessExitDoNotHang, calculateSha1 } from 'playwright-core/lib/utils'; import { stopProfiling, startProfiling, gracefullyProcessExitDoNotHang } from 'playwright-core/lib/utils';
import { serializeError } from './util'; import { serializeError } from './util';
import { showHTMLReport } from './reporters/html'; import { showHTMLReport } from './reporters/html';
import { createMergedReport } from './reporters/merge'; import { createMergedReport } from './reporters/merge';
@ -183,7 +183,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
if (!config) if (!config)
return; return;
config.commandHash = computeCommandHash(args, opts);
config.cliArgs = args; config.cliArgs = args;
config.cliGrep = opts.grep as string | undefined; config.cliGrep = opts.grep as string | undefined;
config.cliGrepInvert = opts.grepInvert as string | undefined; config.cliGrepInvert = opts.grepInvert as string | undefined;
@ -202,19 +201,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
gracefullyProcessExitDoNotHang(exitCode); gracefullyProcessExitDoNotHang(exitCode);
} }
function computeCommandHash(args: string[], opts: { [key: string]: any }) {
const command = {
args,
opts: {} as any
};
// Omit shard from the hash, shard number is added explicitely at the end of the blob report file name.
const keys = Object.keys(opts).filter(k => k !== 'shard');
if (!keys.length && !args.length)
return '';
keys.sort().forEach(key => command.opts[key] = opts[key]);
return calculateSha1(JSON.stringify(command)).substring(0, 7);
}
async function runTestServer(opts: { [key: string]: any }) { async function runTestServer(opts: { [key: string]: any }) {
const host = opts.host || 'localhost'; const host = opts.host || 'localhost';
const port = opts.port ? +opts.port : 0; const port = opts.port ? +opts.port : 0;

4
packages/playwright/src/reporters/blob.ts
View file

@ -16,7 +16,7 @@
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { ManualPromise, calculateSha1, createGuid, getUserAgent, removeFolders } from 'playwright-core/lib/utils'; import { ManualPromise, calculateSha1, createGuid, getUserAgent, removeFolders, sanitizeForFilePath } from 'playwright-core/lib/utils';
import { mime } from 'playwright-core/lib/utilsBundle'; import { mime } from 'playwright-core/lib/utilsBundle';
import { Readable } from 'stream'; import { Readable } from 'stream';
import type { EventEmitter } from 'events'; import type { EventEmitter } from 'events';
@ -114,7 +114,7 @@ export class BlobReporter extends TeleReporterEmitter {
return this._options.fileName; return this._options.fileName;
let reportName = 'report'; let reportName = 'report';
if (this._options._commandHash) if (this._options._commandHash)
reportName += '-' + this._options._commandHash; reportName += '-' + sanitizeForFilePath(this._options._commandHash);
if (config.shard) { if (config.shard) {
const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0'); const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0');
reportName = `${reportName}-${paddedNumber}`; reportName = `${reportName}-${paddedNumber}`;

20
packages/playwright/src/runner/reporters.ts
View file

@ -32,6 +32,7 @@ import { loadReporter } from './loadUtils';
import { BlobReporter } from '../reporters/blob'; import { BlobReporter } from '../reporters/blob';
import type { ReporterDescription } from '../../types/test'; import type { ReporterDescription } from '../../types/test';
import { type ReporterV2, wrapReporterAsV2 } from '../reporters/reporterV2'; import { type ReporterV2, wrapReporterAsV2 } from '../reporters/reporterV2';
import { calculateSha1 } from 'playwright-core/lib/utils';
export async function createReporters(config: FullConfigInternal, mode: 'list' | 'test' | 'merge', isTestServer: boolean, descriptions?: ReporterDescription[]): Promise<ReporterV2[]> { export async function createReporters(config: FullConfigInternal, mode: 'list' | 'test' | 'merge', isTestServer: boolean, descriptions?: ReporterDescription[]): Promise<ReporterV2[]> {
const defaultReporters: { [key in BuiltInReporter]: new(arg: any) => ReporterV2 } = { const defaultReporters: { [key in BuiltInReporter]: new(arg: any) => ReporterV2 } = {
@ -90,10 +91,27 @@ function reporterOptions(config: FullConfigInternal, mode: 'list' | 'test' | 'me
configDir: config.configDir, configDir: config.configDir,
_mode: mode, _mode: mode,
_isTestServer: isTestServer, _isTestServer: isTestServer,
_commandHash: config.commandHash, _commandHash: computeCommandHash(config),
}; };
} }
function computeCommandHash(config: FullConfigInternal) {
let parts = [];
// Include project names for readability.
if (config.cliProjectFilter)
parts.push(...config.cliProjectFilter);
const command = {} as any;
if (config.cliArgs.length)
command.cliArgs = config.cliArgs;
if (config.cliGrep)
command.cliGrep = config.cliGrep;
if (config.cliGrepInvert)
command.cliGrepInvert = config.cliGrepInvert;
if (Object.keys(command).length)
parts.push(calculateSha1(JSON.stringify(command)).substring(0, 7));
return parts.join('-');
}
class ListModeReporter extends EmptyReporter { class ListModeReporter extends EmptyReporter {
private config!: FullConfig; private config!: FullConfig;

72
tests/playwright-test/reporter-blob.spec.ts
View file

@ -274,7 +274,7 @@ test('should merge blob into blob', async ({ runInlineTest, mergeReports, showRe
{ {
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual(['report-873189c-1.zip', 'report-873189c-2.zip']); expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
const { exitCode } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'blob'] }); const { exitCode } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'blob'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
} }
@ -440,7 +440,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/, /report-.*-3.zip/].map(r => expect.stringMatching(r))); expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip', 'report-3.zip']);
const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] }); const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -519,7 +519,7 @@ test('should print progress', async ({ runInlineTest, mergeReports }) => {
await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { additionalArgs: ['--reporter', 'html'] }); const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -570,7 +570,7 @@ test('preserve attachments', async ({ runInlineTest, mergeReports, showReport, p
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual(['report-873189c-1.zip']); expect(reportFiles).toEqual(['report-1.zip']);
const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -633,7 +633,7 @@ test('generate html with attachment urls', async ({ runInlineTest, mergeReports,
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual(['report-873189c-1.zip']); expect(reportFiles).toEqual(['report-1.zip']);
const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -707,7 +707,7 @@ test('resource names should not clash between runs', async ({ runInlineTest, sho
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -780,7 +780,7 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-.*-1.zip/)]); expect(reportFiles).toEqual(['report-1.zip']);
const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html,line'] }); const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html,line'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -841,7 +841,7 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] }); const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -988,7 +988,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-3.zip/].map(r => expect.stringMatching(r))); expect(reportFiles).toEqual(['report-1.zip', 'report-3.zip']);
const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js'), '-c', test.info().outputPath('merge.config.ts')] }); const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js'), '-c', test.info().outputPath('merge.config.ts')] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
const json = JSON.parse(fs.readFileSync(test.info().outputPath('config.json')).toString()); const json = JSON.parse(fs.readFileSync(test.info().outputPath('config.json')).toString());
@ -1144,7 +1144,7 @@ test('preserve steps in html report', async ({ runInlineTest, mergeReports, show
await runInlineTest(files); await runInlineTest(files);
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual(['report-873189c.zip']); expect(reportFiles).toEqual(['report.zip']);
// Run merger in a different directory to make sure relative paths will not be resolved // Run merger in a different directory to make sure relative paths will not be resolved
// relative to the current directory. // relative to the current directory.
const mergeCwd = test.info().outputPath('foo'); const mergeCwd = test.info().outputPath('foo');
@ -1277,9 +1277,9 @@ test('blob report should include version', async ({ runInlineTest }) => {
// CI/1 is a part of user agent string, make sure it matches in the nested test runner. // CI/1 is a part of user agent string, make sure it matches in the nested test runner.
await runInlineTest(files, undefined, { CI: process.env.CI }); await runInlineTest(files, undefined, { CI: process.env.CI });
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
expect(reportFiles).toEqual(['report-873189c.zip']); expect(reportFiles).toEqual(['report.zip']);
const events = await extractReport(test.info().outputPath('blob-report', 'report-873189c.zip'), test.info().outputPath('tmp')); const events = await extractReport(test.info().outputPath('blob-report', 'report.zip'), test.info().outputPath('tmp'));
const metadataEvent = events.find(e => e.method === 'onBlobReportMetadata'); const metadataEvent = events.find(e => e.method === 'onBlobReportMetadata');
expect(metadataEvent.params.version).toBe(2); expect(metadataEvent.params.version).toBe(2);
expect(metadataEvent.params.userAgent).toBe(getUserAgent()); expect(metadataEvent.params.userAgent).toBe(getUserAgent());
@ -1330,7 +1330,7 @@ test('merge-reports should throw if report version is from the future', async ({
await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' }); await runInlineTest(files, { shard: `2/2` }, { PWTEST_BLOB_DO_NOT_REMOVE: '1' });
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
expect(reportFiles).toEqual([/report-.*-1.zip/, /report-.*-2.zip/].map(r => expect.stringMatching(r))); expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']);
// Extract report and modify version. // Extract report and modify version.
const reportZipFile = test.info().outputPath('blob-report', reportFiles[1]); const reportZipFile = test.info().outputPath('blob-report', reportFiles[1]);
@ -1346,7 +1346,7 @@ test('merge-reports should throw if report version is from the future', async ({
const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(1); expect(exitCode).toBe(1);
expect(output).toContain(`Error: Blob report report-873189c-2.zip was created with a newer version of Playwright.`); expect(output).toContain(`Error: Blob report report-2.zip was created with a newer version of Playwright.`);
}); });
@ -1383,7 +1383,7 @@ test('should merge blob reports with same name', async ({ runInlineTest, mergeRe
` `
}; };
await runInlineTest(files); await runInlineTest(files);
const reportZip = test.info().outputPath('blob-report', 'report-873189c.zip'); const reportZip = test.info().outputPath('blob-report', 'report.zip');
const allReportsDir = test.info().outputPath('all-blob-reports'); const allReportsDir = test.info().outputPath('all-blob-reports');
await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-1.zip')); await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-1.zip'));
await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-2.zip')); await fs.promises.cp(reportZip, path.join(allReportsDir, 'report-2.zip'));
@ -1485,8 +1485,8 @@ test('merge reports with different rootDirs', async ({ runInlineTest, mergeRepor
await runInlineTest(files2, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir2/playwright.config.ts')] }); await runInlineTest(files2, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir2/playwright.config.ts')] });
const allReportsDir = test.info().outputPath('all-blob-reports'); const allReportsDir = test.info().outputPath('all-blob-reports');
await fs.promises.cp(test.info().outputPath('dir1', 'blob-report', 'report-546b561.zip'), path.join(allReportsDir, 'report-1.zip')); await fs.promises.cp(test.info().outputPath('dir1', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-1.zip'));
await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report-f033fe4.zip'), path.join(allReportsDir, 'report-2.zip')); await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-2.zip'));
{ {
const { exitCode, output } = await mergeReports(allReportsDir); const { exitCode, output } = await mergeReports(allReportsDir);
@ -1601,7 +1601,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn
await fs.promises.mkdir(allReportsDir, { recursive: true }); await fs.promises.mkdir(allReportsDir, { recursive: true });
// Extract report and change path separators. // Extract report and change path separators.
const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report-e4e2def.zip'); const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report.zip');
const events = await extractReport(reportZipFile, test.info().outputPath('tmp')); const events = await extractReport(reportZipFile, test.info().outputPath('tmp'));
events.forEach(patchPathSeparators); events.forEach(patchPathSeparators);
@ -1610,7 +1610,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn
await zipReport(events, report1); await zipReport(events, report1);
// Copy second report as is. // Copy second report as is.
await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report-b754021.zip'), path.join(allReportsDir, 'report-2.zip')); await fs.promises.cp(test.info().outputPath('dir2', 'blob-report', 'report.zip'), path.join(allReportsDir, 'report-2.zip'));
{ {
const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] }); const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] });
@ -1655,7 +1655,7 @@ test('merge reports without --config preserves path separators', async ({ runInl
await fs.promises.mkdir(allReportsDir, { recursive: true }); await fs.promises.mkdir(allReportsDir, { recursive: true });
// Extract report and change path separators. // Extract report and change path separators.
const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report-6fabe8b.zip'); const reportZipFile = test.info().outputPath('dir1', 'blob-report', 'report.zip');
const events = await extractReport(reportZipFile, test.info().outputPath('tmp')); const events = await extractReport(reportZipFile, test.info().outputPath('tmp'));
events.forEach(patchPathSeparators); events.forEach(patchPathSeparators);
@ -1795,3 +1795,35 @@ test('preserve static annotations when tests did not run', async ({ runInlineTes
await page.goBack(); await page.goBack();
} }
}); });
test('project filter in report name', async ({ runInlineTest }) => {
const files = {
'playwright.config.ts': `
module.exports = {
reporter: 'blob',
projects: [
{ name: 'foo' },
{ name: 'b%/\\ar' },
{ name: 'baz' },
]
};
`,
'a.test.js': `
import { test, expect } from '@playwright/test';
test('math 1 @smoke', async ({}) => {});
`,
};
const reportDir = test.info().outputPath('blob-report');
{
await runInlineTest(files, { shard: `2/2`, project: 'foo' });
const reportFiles = await fs.promises.readdir(reportDir);
expect(reportFiles.sort()).toEqual(['report-foo-2.zip']);
}
{
await runInlineTest(files, { shard: `1/2`, project: 'foo,b*r', grep: 'smoke' });
const reportFiles = await fs.promises.readdir(reportDir);
expect(reportFiles.sort()).toEqual(['report-foo-b-r-6d9d49e-1.zip']);
}
});