Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore: implement repeat last run (#20727)

Browse source
This commit is contained in:
Pavel Feldman 2023-02-07 15:56:39 -08:00 committed by GitHub
parent 1b941bcf2e
commit 4259d4e1d6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 59 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/playwright-test/src/cli.ts
View file

@ -167,8 +167,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
config._internal.passWithNoTests = !!opts.passWithNoTests; config._internal.passWithNoTests = !!opts.passWithNoTests;
const runner = new Runner(config); const runner = new Runner(config);
if (opts.watch)
process.stdout.write('\x1Bc');
const status = await runner.runAllTests(!!opts.watch); const status = await runner.runAllTests(!!opts.watch);
await stopProfiling(undefined); await stopProfiling(undefined);

26
packages/playwright-test/src/reporters/base.ts
View file

@ -123,7 +123,7 @@ export class BaseReporter implements Reporter {
protected generateStartingMessage() { protected generateStartingMessage() {
const jobs = Math.min(this.config.workers, this.config._internal.maxConcurrentTestGroups); const jobs = Math.min(this.config.workers, this.config._internal.maxConcurrentTestGroups);
const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : '';
return `\nRunning ${this.totalTestCount} test${this.totalTestCount !== 1 ? 's' : ''} using ${jobs} worker${jobs !== 1 ? 's' : ''}${shardDetails}`; return '\n' + colors.dim('Running ') + this.totalTestCount + colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}`);
} }
protected getSlowTests(): [string, number][] { protected getSlowTests(): [string, number][] {
@ -258,7 +258,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
const retryLines = []; const retryLines = [];
if (result.retry) { if (result.retry) {
retryLines.push(''); retryLines.push('');
retryLines.push(colors.gray(pad(` Retry #${result.retry}`, '-'))); retryLines.push(colors.gray(separator(` Retry #${result.retry}`)));
} }
resultLines.push(...retryLines); resultLines.push(...retryLines);
resultLines.push(...errors.map(error => '\n' + error.message)); resultLines.push(...errors.map(error => '\n' + error.message));
@ -269,7 +269,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
if (!attachment.path && !hasPrintableContent) if (!attachment.path && !hasPrintableContent)
continue; continue;
resultLines.push(''); resultLines.push('');
resultLines.push(colors.cyan(pad(` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`, '-'))); resultLines.push(colors.cyan(separator(` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`)));
if (attachment.path) { if (attachment.path) {
const relativePath = path.relative(process.cwd(), attachment.path); const relativePath = path.relative(process.cwd(), attachment.path);
resultLines.push(colors.cyan(` ${relativePath}`)); resultLines.push(colors.cyan(` ${relativePath}`));
@ -288,7 +288,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
resultLines.push(colors.cyan(` ${text}`)); resultLines.push(colors.cyan(` ${text}`));
} }
} }
resultLines.push(colors.cyan(pad(' ', '-'))); resultLines.push(colors.cyan(separator(' ')));
} }
} }
const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[]; const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[];
@ -300,7 +300,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
return text; return text;
}).join(''); }).join('');
resultLines.push(''); resultLines.push('');
resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-')); resultLines.push(colors.gray(separator('--- Test output')) + '\n\n' + outputText + '\n' + separator());
} }
for (const error of errors) { for (const error of errors) {
annotations.push({ annotations.push({
@ -370,7 +370,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestS
function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string { function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string {
const title = formatTestTitle(config, test); const title = formatTestTitle(config, test);
const header = `${indent}${index ? index + ') ' : ''}${title}`; const header = `${indent}${index ? index + ') ' : ''}${title}`;
return pad(header, '='); return separator(header);
} }
export function formatError(config: FullConfig, error: TestError, highlightCode: boolean, file?: string): ErrorDetails { export function formatError(config: FullConfig, error: TestError, highlightCode: boolean, file?: string): ErrorDetails {
@ -416,10 +416,11 @@ export function formatError(config: FullConfig, error: TestError, highlightCode:
}; };
} }
function pad(line: string, char: string): string { export function separator(text: string = ''): string {
if (line) if (text)
line += ' '; text += ' ';
return line + colors.gray(char.repeat(Math.max(0, 100 - line.length))); const columns = Math.min(100, process.stdout?.columns || 100);
return text + colors.gray('─'.repeat(Math.max(0, columns - text.length)));
} }
function indent(lines: string, tab: string) { function indent(lines: string, tab: string) {
@ -485,8 +486,3 @@ function fitToWidth(line: string, width: number, prefix?: string): string {
function belongsToNodeModules(file: string) { function belongsToNodeModules(file: string) {
return file.includes(`${path.sep}node_modules${path.sep}`); return file.includes(`${path.sep}node_modules${path.sep}`);
} }
export function separator(): string {
const columns = process.stdout?.columns || 30;
return colors.dim('⎯'.repeat(Math.min(100, columns)));
}

3
packages/playwright-test/src/runner/reporters.ts
View file

@ -120,8 +120,7 @@ export class WatchModeReporter extends ListReporter {
const lines: string[] = []; const lines: string[] = [];
const sep = separator(); const sep = separator();
lines.push('\x1Bc' + sep); lines.push('\x1Bc' + sep);
lines.push(`${tokens.join(' ')}`); lines.push(`${tokens.join(' ')}` + super.generateStartingMessage());
lines.push(sep + super.generateStartingMessage());
return lines.join('\n'); return lines.join('\n');
} }
} }

1
packages/playwright-test/src/runner/tasks.ts
View file

@ -161,6 +161,7 @@ function createLoadTask(mode: 'out-of-process' | 'in-process', projectsToIgnore
function createTestGroupsTask(): Task<TaskRunnerState> { function createTestGroupsTask(): Task<TaskRunnerState> {
return async context => { return async context => {
const { config, rootSuite, reporter } = context; const { config, rootSuite, reporter } = context;
context.config._internal.maxConcurrentTestGroups = 0;
for (const phase of buildPhases(rootSuite!.suites)) { for (const phase of buildPhases(rootSuite!.suites)) {
// Go over the phases, for each phase create list of task groups. // Go over the phases, for each phase create list of task groups.
const projects: ProjectWithTestGroups[] = []; const projects: ProjectWithTestGroups[] = [];

36
packages/playwright-test/src/runner/watchMode.ts
View file

@ -70,6 +70,7 @@ export async function runWatchModeLoop(config: FullConfigInternal, failedTests:
const originalCliArgs = config._internal.cliArgs; const originalCliArgs = config._internal.cliArgs;
const originalCliGrep = config._internal.cliGrep; const originalCliGrep = config._internal.cliGrep;
let lastRun: { type: 'changed' | 'regular' | 'failed', failedTestIds?: Set<string>, dirtyFiles?: Set<string> } = { type: 'regular' };
const fsWatcher = new FSWatcher(projectClosure.map(p => p.testDir)); const fsWatcher = new FSWatcher(projectClosure.map(p => p.testDir));
while (true) { while (true) {
@ -87,17 +88,23 @@ Waiting for file changes. Press ${colors.bold('h')} for help or ${colors.bold('q
readCommandPromise.resolve('changed'); readCommandPromise.resolve('changed');
const command = await readCommandPromise; const command = await readCommandPromise;
if (command === 'changed') { if (command === 'changed') {
await runChangedTests(config, failedTestIdCollector, projectClosure, fsWatcher.takeDirtyFiles()); const dirtyFiles = fsWatcher.takeDirtyFiles();
await runChangedTests(config, failedTestIdCollector, projectClosure, dirtyFiles);
lastRun = { type: 'changed', dirtyFiles };
continue; continue;
} }
if (command === 'all') { if (command === 'all') {
// All means reset filters. // All means reset filters.
config._internal.cliArgs = originalCliArgs; config._internal.cliArgs = originalCliArgs;
config._internal.cliGrep = originalCliGrep; config._internal.cliGrep = originalCliGrep;
await runTests(config, failedTestIdCollector); await runTests(config, failedTestIdCollector);
lastRun = { type: 'regular' };
continue; continue;
} }
if (command === 'file') { if (command === 'file') {
const { filePattern } = await enquirer.prompt<{ filePattern: string }>({ const { filePattern } = await enquirer.prompt<{ filePattern: string }>({
type: 'text', type: 'text',
@ -110,8 +117,10 @@ Waiting for file changes. Press ${colors.bold('h')} for help or ${colors.bold('q
else else
config._internal.cliArgs = []; config._internal.cliArgs = [];
await runTests(config, failedTestIdCollector); await runTests(config, failedTestIdCollector);
lastRun = { type: 'regular' };
continue; continue;
} }
if (command === 'grep') { if (command === 'grep') {
const { testPattern } = await enquirer.prompt<{ testPattern: string }>({ const { testPattern } = await enquirer.prompt<{ testPattern: string }>({
type: 'text', type: 'text',
@ -124,19 +133,36 @@ Waiting for file changes. Press ${colors.bold('h')} for help or ${colors.bold('q
else else
config._internal.cliGrep = undefined; config._internal.cliGrep = undefined;
await runTests(config, failedTestIdCollector); await runTests(config, failedTestIdCollector);
lastRun = { type: 'regular' };
continue; continue;
} }
if (command === 'failed') { if (command === 'failed') {
config._internal.testIdMatcher = id => failedTestIdCollector.has(id); config._internal.testIdMatcher = id => failedTestIdCollector.has(id);
try { const failedTestIds = new Set(failedTestIdCollector);
await runTests(config, failedTestIdCollector);
config._internal.testIdMatcher = undefined;
lastRun = { type: 'failed', failedTestIds };
continue;
}
if (command === 'repeat') {
if (lastRun.type === 'regular') {
await runTests(config, failedTestIdCollector);
continue;
} else if (lastRun.type === 'changed') {
await runChangedTests(config, failedTestIdCollector, projectClosure, lastRun.dirtyFiles!);
} else if (lastRun.type === 'failed') {
config._internal.testIdMatcher = id => lastRun.failedTestIds!.has(id);
await runTests(config, failedTestIdCollector); await runTests(config, failedTestIdCollector);
} finally {
config._internal.testIdMatcher = undefined; config._internal.testIdMatcher = undefined;
} }
continue; continue;
} }
if (command === 'exit') if (command === 'exit')
return 'passed'; return 'passed';
if (command === 'interrupted') if (command === 'interrupted')
return 'interrupted'; return 'interrupted';
} }
@ -254,6 +280,7 @@ ${commands.map(i => ' ' + colors.bold(i[0]) + `: ${i[1]}`).join('\n')}
case 'p': result.resolve('file'); break; case 'p': result.resolve('file'); break;
case 't': result.resolve('grep'); break; case 't': result.resolve('grep'); break;
case 'f': result.resolve('failed'); break; case 'f': result.resolve('failed'); break;
case 'r': result.resolve('repeat'); break;
} }
}; };
@ -267,11 +294,12 @@ ${commands.map(i => ' ' + colors.bold(i[0]) + `: ${i[1]}`).join('\n')}
return result; return result;
} }
type Command = 'all' | 'failed' | 'changed' | 'file' | 'grep' | 'exit' | 'interrupted'; type Command = 'all' | 'failed' | 'repeat' | 'changed' | 'file' | 'grep' | 'exit' | 'interrupted';
const commands = [ const commands = [
['a', 'rerun all tests'], ['a', 'rerun all tests'],
['f', 'rerun only failed tests'], ['f', 'rerun only failed tests'],
['r', 'repeat last run'],
['p', 'filter by a filename'], ['p', 'filter by a filename'],
['t', 'filter by a test name regex pattern'], ['t', 'filter by a test name regex pattern'],
['q', 'quit'], ['q', 'quit'],

16
tests/playwright-test/reporter-attachment.spec.ts
View file

@ -32,9 +32,9 @@ test('render text attachment', async ({ runInlineTest }) => {
`, `,
}, { reporter: 'line' }); }, { reporter: 'line' });
const text = result.output; const text = result.output;
expect(text).toContain(' attachment #1: attachment (text/plain) ---------------------------------------------------------'); expect(text).toContain(' attachment #1: attachment (text/plain) ─────────────────────────────────────────────────────────');
expect(text).toContain(' Hello world'); expect(text).toContain(' Hello world');
expect(text).toContain(' ------------------------------------------------------------------------------------------------'); expect(text).toContain(' ────────────────────────────────────────────────────────────────────────────────────────────────');
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
}); });
@ -53,9 +53,9 @@ test('render screenshot attachment', async ({ runInlineTest }) => {
`, `,
}, { reporter: 'line' }); }, { reporter: 'line' });
const text = result.output.replace(/\\/g, '/'); const text = result.output.replace(/\\/g, '/');
expect(text).toContain(' attachment #1: screenshot (image/png) ----------------------------------------------------------'); expect(text).toContain(' attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────');
expect(text).toContain(' test-results/a-one/some/path.png'); expect(text).toContain(' test-results/a-one/some/path.png');
expect(text).toContain(' ------------------------------------------------------------------------------------------------'); expect(text).toContain(' ────────────────────────────────────────────────────────────────────────────────────────────────');
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
}); });
@ -74,10 +74,10 @@ test('render trace attachment', async ({ runInlineTest }) => {
`, `,
}, { reporter: 'line' }); }, { reporter: 'line' });
const text = result.output.replace(/\\/g, '/'); const text = result.output.replace(/\\/g, '/');
expect(text).toContain(' attachment #1: trace (application/zip) ---------------------------------------------------------'); expect(text).toContain(' attachment #1: trace (application/zip) ─────────────────────────────────────────────────────────');
expect(text).toContain(' test-results/a-one/trace.zip'); expect(text).toContain(' test-results/a-one/trace.zip');
expect(text).toContain('npx playwright show-trace test-results/a-one/trace.zip'); expect(text).toContain('npx playwright show-trace test-results/a-one/trace.zip');
expect(text).toContain(' ------------------------------------------------------------------------------------------------'); expect(text).toContain(' ────────────────────────────────────────────────────────────────────────────────────────────────');
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
}); });
@ -170,7 +170,7 @@ test(`testInfo.attach allow empty string body`, async ({ runInlineTest }) => {
}); });
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1); expect(result.failed).toBe(1);
expect(result.output).toMatch(/^.*attachment #1: name \(text\/plain\).*\n.*\n.*------/gm); expect(result.output).toMatch(/^.*attachment #1: name \(text\/plain\).*\n.*\n.*──────/gm);
}); });
test(`testInfo.attach allow empty buffer body`, async ({ runInlineTest }) => { test(`testInfo.attach allow empty buffer body`, async ({ runInlineTest }) => {
@ -185,7 +185,7 @@ test(`testInfo.attach allow empty buffer body`, async ({ runInlineTest }) => {
}); });
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1); expect(result.failed).toBe(1);
expect(result.output).toMatch(/^.*attachment #1: name \(text\/plain\).*\n.*\n.*------/gm); expect(result.output).toMatch(/^.*attachment #1: name \(text\/plain\).*\n.*\n.*──────/gm);
}); });
test(`testInfo.attach use name as prefix`, async ({ runInlineTest }) => { test(`testInfo.attach use name as prefix`, async ({ runInlineTest }) => {

6
tests/playwright-test/reporter-github.spec.ts
View file

@ -49,9 +49,9 @@ test('print GitHub annotations for failed tests', async ({ runInlineTest }, test
}, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() }); }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() });
const text = result.output; const text = result.output;
const testPath = relativeFilePath(testInfo.outputPath('a.test.js')); const testPath = relativeFilePath(testInfo.outputPath('a.test.js'));
expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example =======================================================================%0A%0A Retry #1`); expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #1`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example =======================================================================%0A%0A Retry #2`); expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #2`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example =======================================================================%0A%0A Retry #3`); expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #3`);
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
}); });

6
tests/playwright-test/reporter-line.spec.ts
View file

@ -33,9 +33,9 @@ test('render unexpected after retry', async ({ runInlineTest }) => {
expect(text).toContain('1 failed'); expect(text).toContain('1 failed');
expect(text).toContain('1) a.test'); expect(text).toContain('1) a.test');
expect(text).not.toContain('2) a.test'); expect(text).not.toContain('2) a.test');
expect(text).toContain('Retry #1 ----'); expect(text).toContain('Retry #1 ────');
expect(text).toContain('Retry #2 ----'); expect(text).toContain('Retry #2 ────');
expect(text).toContain('Retry #3 ----'); expect(text).toContain('Retry #3 ────');
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(1);
}); });