From 4259d4e1d67042612fdae98a123da12a8cea5c2c Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Tue, 7 Feb 2023 15:56:39 -0800 Subject: [PATCH] chore: implement repeat last run (#20727) --- packages/playwright-test/src/cli.ts | 2 -- .../playwright-test/src/reporters/base.ts | 26 ++++++-------- .../playwright-test/src/runner/reporters.ts | 3 +- packages/playwright-test/src/runner/tasks.ts | 1 + .../playwright-test/src/runner/watchMode.ts | 36 ++++++++++++++++--- .../reporter-attachment.spec.ts | 16 ++++----- tests/playwright-test/reporter-github.spec.ts | 6 ++-- tests/playwright-test/reporter-line.spec.ts | 6 ++-- 8 files changed, 59 insertions(+), 37 deletions(-) diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index f880bfe1a0..7f19b3f40f 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -167,8 +167,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) { config._internal.passWithNoTests = !!opts.passWithNoTests; const runner = new Runner(config); - if (opts.watch) - process.stdout.write('\x1Bc'); const status = await runner.runAllTests(!!opts.watch); await stopProfiling(undefined); diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 0e778d5fd1..5ffdcf0b31 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -123,7 +123,7 @@ export class BaseReporter implements Reporter { protected generateStartingMessage() { const jobs = Math.min(this.config.workers, this.config._internal.maxConcurrentTestGroups); const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; - return `\nRunning ${this.totalTestCount} test${this.totalTestCount !== 1 ? 's' : ''} using ${jobs} worker${jobs !== 1 ? 's' : ''}${shardDetails}`; + return '\n' + colors.dim('Running ') + this.totalTestCount + colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}`); } protected getSlowTests(): [string, number][] { @@ -258,7 +258,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde const retryLines = []; if (result.retry) { retryLines.push(''); - retryLines.push(colors.gray(pad(` Retry #${result.retry}`, '-'))); + retryLines.push(colors.gray(separator(` Retry #${result.retry}`))); } resultLines.push(...retryLines); resultLines.push(...errors.map(error => '\n' + error.message)); @@ -269,7 +269,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde if (!attachment.path && !hasPrintableContent) continue; resultLines.push(''); - resultLines.push(colors.cyan(pad(` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`, '-'))); + resultLines.push(colors.cyan(separator(` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`))); if (attachment.path) { const relativePath = path.relative(process.cwd(), attachment.path); resultLines.push(colors.cyan(` ${relativePath}`)); @@ -288,7 +288,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde resultLines.push(colors.cyan(` ${text}`)); } } - resultLines.push(colors.cyan(pad(' ', '-'))); + resultLines.push(colors.cyan(separator(' '))); } } const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[]; @@ -300,7 +300,7 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde return text; }).join(''); resultLines.push(''); - resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-')); + resultLines.push(colors.gray(separator('--- Test output')) + '\n\n' + outputText + '\n' + separator()); } for (const error of errors) { annotations.push({ @@ -370,7 +370,7 @@ export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestS function formatTestHeader(config: FullConfig, test: TestCase, indent: string, index?: number): string { const title = formatTestTitle(config, test); const header = `${indent}${index ? index + ') ' : ''}${title}`; - return pad(header, '='); + return separator(header); } export function formatError(config: FullConfig, error: TestError, highlightCode: boolean, file?: string): ErrorDetails { @@ -416,10 +416,11 @@ export function formatError(config: FullConfig, error: TestError, highlightCode: }; } -function pad(line: string, char: string): string { - if (line) - line += ' '; - return line + colors.gray(char.repeat(Math.max(0, 100 - line.length))); +export function separator(text: string = ''): string { + if (text) + text += ' '; + const columns = Math.min(100, process.stdout?.columns || 100); + return text + colors.gray('─'.repeat(Math.max(0, columns - text.length))); } function indent(lines: string, tab: string) { @@ -485,8 +486,3 @@ function fitToWidth(line: string, width: number, prefix?: string): string { function belongsToNodeModules(file: string) { return file.includes(`${path.sep}node_modules${path.sep}`); } - -export function separator(): string { - const columns = process.stdout?.columns || 30; - return colors.dim('⎯'.repeat(Math.min(100, columns))); -} diff --git a/packages/playwright-test/src/runner/reporters.ts b/packages/playwright-test/src/runner/reporters.ts index 1de20189c8..b5cd84f5a3 100644 --- a/packages/playwright-test/src/runner/reporters.ts +++ b/packages/playwright-test/src/runner/reporters.ts @@ -120,8 +120,7 @@ export class WatchModeReporter extends ListReporter { const lines: string[] = []; const sep = separator(); lines.push('\x1Bc' + sep); - lines.push(`${tokens.join(' ')}`); - lines.push(sep + super.generateStartingMessage()); + lines.push(`${tokens.join(' ')}` + super.generateStartingMessage()); return lines.join('\n'); } } diff --git a/packages/playwright-test/src/runner/tasks.ts b/packages/playwright-test/src/runner/tasks.ts index 3cf6149d1e..3e4625bdeb 100644 --- a/packages/playwright-test/src/runner/tasks.ts +++ b/packages/playwright-test/src/runner/tasks.ts @@ -161,6 +161,7 @@ function createLoadTask(mode: 'out-of-process' | 'in-process', projectsToIgnore function createTestGroupsTask(): Task { return async context => { const { config, rootSuite, reporter } = context; + context.config._internal.maxConcurrentTestGroups = 0; for (const phase of buildPhases(rootSuite!.suites)) { // Go over the phases, for each phase create list of task groups. const projects: ProjectWithTestGroups[] = []; diff --git a/packages/playwright-test/src/runner/watchMode.ts b/packages/playwright-test/src/runner/watchMode.ts index e87b022b42..d8d4be7429 100644 --- a/packages/playwright-test/src/runner/watchMode.ts +++ b/packages/playwright-test/src/runner/watchMode.ts @@ -70,6 +70,7 @@ export async function runWatchModeLoop(config: FullConfigInternal, failedTests: const originalCliArgs = config._internal.cliArgs; const originalCliGrep = config._internal.cliGrep; + let lastRun: { type: 'changed' | 'regular' | 'failed', failedTestIds?: Set, dirtyFiles?: Set } = { type: 'regular' }; const fsWatcher = new FSWatcher(projectClosure.map(p => p.testDir)); while (true) { @@ -87,17 +88,23 @@ Waiting for file changes. Press ${colors.bold('h')} for help or ${colors.bold('q readCommandPromise.resolve('changed'); const command = await readCommandPromise; + if (command === 'changed') { - await runChangedTests(config, failedTestIdCollector, projectClosure, fsWatcher.takeDirtyFiles()); + const dirtyFiles = fsWatcher.takeDirtyFiles(); + await runChangedTests(config, failedTestIdCollector, projectClosure, dirtyFiles); + lastRun = { type: 'changed', dirtyFiles }; continue; } + if (command === 'all') { // All means reset filters. config._internal.cliArgs = originalCliArgs; config._internal.cliGrep = originalCliGrep; await runTests(config, failedTestIdCollector); + lastRun = { type: 'regular' }; continue; } + if (command === 'file') { const { filePattern } = await enquirer.prompt<{ filePattern: string }>({ type: 'text', @@ -110,8 +117,10 @@ Waiting for file changes. Press ${colors.bold('h')} for help or ${colors.bold('q else config._internal.cliArgs = []; await runTests(config, failedTestIdCollector); + lastRun = { type: 'regular' }; continue; } + if (command === 'grep') { const { testPattern } = await enquirer.prompt<{ testPattern: string }>({ type: 'text', @@ -124,19 +133,36 @@ Waiting for file changes. Press ${colors.bold('h')} for help or ${colors.bold('q else config._internal.cliGrep = undefined; await runTests(config, failedTestIdCollector); + lastRun = { type: 'regular' }; continue; } + if (command === 'failed') { config._internal.testIdMatcher = id => failedTestIdCollector.has(id); - try { + const failedTestIds = new Set(failedTestIdCollector); + await runTests(config, failedTestIdCollector); + config._internal.testIdMatcher = undefined; + lastRun = { type: 'failed', failedTestIds }; + continue; + } + + if (command === 'repeat') { + if (lastRun.type === 'regular') { + await runTests(config, failedTestIdCollector); + continue; + } else if (lastRun.type === 'changed') { + await runChangedTests(config, failedTestIdCollector, projectClosure, lastRun.dirtyFiles!); + } else if (lastRun.type === 'failed') { + config._internal.testIdMatcher = id => lastRun.failedTestIds!.has(id); await runTests(config, failedTestIdCollector); - } finally { config._internal.testIdMatcher = undefined; } continue; } + if (command === 'exit') return 'passed'; + if (command === 'interrupted') return 'interrupted'; } @@ -254,6 +280,7 @@ ${commands.map(i => ' ' + colors.bold(i[0]) + `: ${i[1]}`).join('\n')} case 'p': result.resolve('file'); break; case 't': result.resolve('grep'); break; case 'f': result.resolve('failed'); break; + case 'r': result.resolve('repeat'); break; } }; @@ -267,11 +294,12 @@ ${commands.map(i => ' ' + colors.bold(i[0]) + `: ${i[1]}`).join('\n')} return result; } -type Command = 'all' | 'failed' | 'changed' | 'file' | 'grep' | 'exit' | 'interrupted'; +type Command = 'all' | 'failed' | 'repeat' | 'changed' | 'file' | 'grep' | 'exit' | 'interrupted'; const commands = [ ['a', 'rerun all tests'], ['f', 'rerun only failed tests'], + ['r', 'repeat last run'], ['p', 'filter by a filename'], ['t', 'filter by a test name regex pattern'], ['q', 'quit'], diff --git a/tests/playwright-test/reporter-attachment.spec.ts b/tests/playwright-test/reporter-attachment.spec.ts index fe48901e97..48fc59aaa6 100644 --- a/tests/playwright-test/reporter-attachment.spec.ts +++ b/tests/playwright-test/reporter-attachment.spec.ts @@ -32,9 +32,9 @@ test('render text attachment', async ({ runInlineTest }) => { `, }, { reporter: 'line' }); const text = result.output; - expect(text).toContain(' attachment #1: attachment (text/plain) ---------------------------------------------------------'); + expect(text).toContain(' attachment #1: attachment (text/plain) ─────────────────────────────────────────────────────────'); expect(text).toContain(' Hello world'); - expect(text).toContain(' ------------------------------------------------------------------------------------------------'); + expect(text).toContain(' ────────────────────────────────────────────────────────────────────────────────────────────────'); expect(result.exitCode).toBe(1); }); @@ -53,9 +53,9 @@ test('render screenshot attachment', async ({ runInlineTest }) => { `, }, { reporter: 'line' }); const text = result.output.replace(/\\/g, '/'); - expect(text).toContain(' attachment #1: screenshot (image/png) ----------------------------------------------------------'); + expect(text).toContain(' attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────'); expect(text).toContain(' test-results/a-one/some/path.png'); - expect(text).toContain(' ------------------------------------------------------------------------------------------------'); + expect(text).toContain(' ────────────────────────────────────────────────────────────────────────────────────────────────'); expect(result.exitCode).toBe(1); }); @@ -74,10 +74,10 @@ test('render trace attachment', async ({ runInlineTest }) => { `, }, { reporter: 'line' }); const text = result.output.replace(/\\/g, '/'); - expect(text).toContain(' attachment #1: trace (application/zip) ---------------------------------------------------------'); + expect(text).toContain(' attachment #1: trace (application/zip) ─────────────────────────────────────────────────────────'); expect(text).toContain(' test-results/a-one/trace.zip'); expect(text).toContain('npx playwright show-trace test-results/a-one/trace.zip'); - expect(text).toContain(' ------------------------------------------------------------------------------------------------'); + expect(text).toContain(' ────────────────────────────────────────────────────────────────────────────────────────────────'); expect(result.exitCode).toBe(1); }); @@ -170,7 +170,7 @@ test(`testInfo.attach allow empty string body`, async ({ runInlineTest }) => { }); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); - expect(result.output).toMatch(/^.*attachment #1: name \(text\/plain\).*\n.*\n.*------/gm); + expect(result.output).toMatch(/^.*attachment #1: name \(text\/plain\).*\n.*\n.*──────/gm); }); test(`testInfo.attach allow empty buffer body`, async ({ runInlineTest }) => { @@ -185,7 +185,7 @@ test(`testInfo.attach allow empty buffer body`, async ({ runInlineTest }) => { }); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); - expect(result.output).toMatch(/^.*attachment #1: name \(text\/plain\).*\n.*\n.*------/gm); + expect(result.output).toMatch(/^.*attachment #1: name \(text\/plain\).*\n.*\n.*──────/gm); }); test(`testInfo.attach use name as prefix`, async ({ runInlineTest }) => { diff --git a/tests/playwright-test/reporter-github.spec.ts b/tests/playwright-test/reporter-github.spec.ts index d7c8db6076..5b39b35479 100644 --- a/tests/playwright-test/reporter-github.spec.ts +++ b/tests/playwright-test/reporter-github.spec.ts @@ -49,9 +49,9 @@ test('print GitHub annotations for failed tests', async ({ runInlineTest }, test }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() }); const text = result.output; const testPath = relativeFilePath(testInfo.outputPath('a.test.js')); - expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example =======================================================================%0A%0A Retry #1`); - expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example =======================================================================%0A%0A Retry #2`); - expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example =======================================================================%0A%0A Retry #3`); + expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #1`); + expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #2`); + expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #3`); expect(result.exitCode).toBe(1); }); diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.ts index 38cac11449..6af0ae916a 100644 --- a/tests/playwright-test/reporter-line.spec.ts +++ b/tests/playwright-test/reporter-line.spec.ts @@ -33,9 +33,9 @@ test('render unexpected after retry', async ({ runInlineTest }) => { expect(text).toContain('1 failed'); expect(text).toContain('1) a.test'); expect(text).not.toContain('2) a.test'); - expect(text).toContain('Retry #1 ----'); - expect(text).toContain('Retry #2 ----'); - expect(text).toContain('Retry #3 ----'); + expect(text).toContain('Retry #1 ────'); + expect(text).toContain('Retry #2 ────'); + expect(text).toContain('Retry #3 ────'); expect(result.exitCode).toBe(1); });