diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index 1c76aab5e6..f5f8d39809 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -146,6 +146,8 @@ async function runTests(args: string[], opts: { [key: string]: any }) { let status: FullResult['status']; if (opts.ui) status = await runner.uiAllTests(); + else if (process.env.PWTEST_WATCH) + status = await runner.watchAllTests(); else status = await runner.runAllTests(); await stopProfiling('runner'); diff --git a/packages/playwright-test/src/runner/reporters.ts b/packages/playwright-test/src/runner/reporters.ts index 44b5e737e2..cb8ee06c0e 100644 --- a/packages/playwright-test/src/runner/reporters.ts +++ b/packages/playwright-test/src/runner/reporters.ts @@ -31,7 +31,7 @@ import type { FullConfigInternal } from '../common/types'; import { loadReporter } from './loadUtils'; import type { BuiltInReporter } from '../common/configLoader'; -export async function createReporter(config: FullConfigInternal, mode: 'list' | 'run' | 'ui', additionalReporters: Reporter[] = []): Promise { +export async function createReporter(config: FullConfigInternal, mode: 'list' | 'watch' | 'run' | 'ui', additionalReporters: Reporter[] = []): Promise { const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = { dot: mode === 'list' ? ListModeReporter : DotReporter, line: mode === 'list' ? ListModeReporter : LineReporter, @@ -43,19 +43,23 @@ export async function createReporter(config: FullConfigInternal, mode: 'list' | html: mode === 'ui' ? LineReporter : HtmlReporter, }; const reporters: Reporter[] = []; - for (const r of config.reporter) { - const [name, arg] = r; - if (name in defaultReporters) { - reporters.push(new defaultReporters[name as keyof typeof defaultReporters](arg)); - } else { - const reporterConstructor = await loadReporter(config, name); - reporters.push(new reporterConstructor(arg)); + if (mode === 'watch') { + reporters.push(new ListReporter()); + } else { + for (const r of config.reporter) { + const [name, arg] = r; + if (name in defaultReporters) { + reporters.push(new defaultReporters[name as keyof typeof defaultReporters](arg)); + } else { + const reporterConstructor = await loadReporter(config, name); + reporters.push(new reporterConstructor(arg)); + } + } + reporters.push(...additionalReporters); + if (process.env.PW_TEST_REPORTER) { + const reporterConstructor = await loadReporter(config, process.env.PW_TEST_REPORTER); + reporters.push(new reporterConstructor()); } - } - reporters.push(...additionalReporters); - if (process.env.PW_TEST_REPORTER) { - const reporterConstructor = await loadReporter(config, process.env.PW_TEST_REPORTER); - reporters.push(new reporterConstructor()); } const someReporterPrintsToStdio = reporters.some(r => { diff --git a/packages/playwright-test/src/runner/runner.ts b/packages/playwright-test/src/runner/runner.ts index 5d748c59ac..5173428c1f 100644 --- a/packages/playwright-test/src/runner/runner.ts +++ b/packages/playwright-test/src/runner/runner.ts @@ -24,6 +24,7 @@ import { createTaskRunner, createTaskRunnerForList } from './tasks'; import type { TaskRunnerState } from './tasks'; import type { FullConfigInternal } from '../common/types'; import { colors } from 'playwright-core/lib/utilsBundle'; +import { runWatchModeLoop } from './watchMode'; import { runUIMode } from './uiMode'; export class Runner { @@ -92,6 +93,12 @@ export class Runner { return status; } + async watchAllTests(): Promise { + const config = this._config; + webServerPluginsForConfig(config).forEach(p => config._internal.plugins.push({ factory: p })); + return await runWatchModeLoop(config); + } + async uiAllTests(): Promise { const config = this._config; webServerPluginsForConfig(config).forEach(p => config._internal.plugins.push({ factory: p })); diff --git a/packages/playwright-test/src/runner/tasks.ts b/packages/playwright-test/src/runner/tasks.ts index 18505d12ea..63396e1347 100644 --- a/packages/playwright-test/src/runner/tasks.ts +++ b/packages/playwright-test/src/runner/tasks.ts @@ -58,13 +58,13 @@ export function createTaskRunner(config: FullConfigInternal, reporter: Multiplex return taskRunner; } -export function createTaskRunnerForGlobalSetup(config: FullConfigInternal, reporter: Multiplexer): TaskRunner { +export function createTaskRunnerForWatchSetup(config: FullConfigInternal, reporter: Multiplexer): TaskRunner { const taskRunner = new TaskRunner(reporter, 0); addGlobalSetupTasks(taskRunner, config); return taskRunner; } -export function createTaskRunnerForUIMode(config: FullConfigInternal, reporter: Multiplexer, projectsToIgnore?: Set, additionalFileMatcher?: Matcher): TaskRunner { +export function createTaskRunnerForWatch(config: FullConfigInternal, reporter: Multiplexer, projectsToIgnore?: Set, additionalFileMatcher?: Matcher): TaskRunner { const taskRunner = new TaskRunner(reporter, 0); taskRunner.addTask('load tests', createLoadTask('out-of-process', true, projectsToIgnore, additionalFileMatcher)); addRunTasks(taskRunner, config); diff --git a/packages/playwright-test/src/runner/uiMode.ts b/packages/playwright-test/src/runner/uiMode.ts index e5641d9b22..aa69930580 100644 --- a/packages/playwright-test/src/runner/uiMode.ts +++ b/packages/playwright-test/src/runner/uiMode.ts @@ -24,7 +24,7 @@ import { Multiplexer } from '../reporters/multiplexer'; import { TeleReporterEmitter } from '../reporters/teleEmitter'; import { createReporter } from './reporters'; import type { TaskRunnerState } from './tasks'; -import { createTaskRunnerForList, createTaskRunnerForUIMode, createTaskRunnerForGlobalSetup } from './tasks'; +import { createTaskRunnerForList, createTaskRunnerForWatch, createTaskRunnerForWatchSetup } from './tasks'; import { chokidar } from '../utilsBundle'; import type { FSWatcher } from 'chokidar'; import { open } from '../utilsBundle'; @@ -67,7 +67,7 @@ class UIMode { async runGlobalSetup(): Promise { const reporter = await createReporter(this._config, 'run'); - const taskRunner = createTaskRunnerForGlobalSetup(this._config, reporter); + const taskRunner = createTaskRunnerForWatchSetup(this._config, reporter); reporter.onConfigure(this._config); const context: TaskRunnerState = { config: this._config, @@ -169,7 +169,7 @@ class UIMode { const runReporter = new TeleReporterEmitter(e => this._dispatchEvent(e)); const reporter = await createReporter(this._config, 'ui', [runReporter]); - const taskRunner = createTaskRunnerForUIMode(this._config, reporter); + const taskRunner = createTaskRunnerForWatch(this._config, reporter); const context: TaskRunnerState = { config: this._config, reporter, phases: [] }; clearCompilationCache(); reporter.onConfigure(this._config); diff --git a/packages/playwright-test/src/runner/watchMode.ts b/packages/playwright-test/src/runner/watchMode.ts new file mode 100644 index 0000000000..df1ca74082 --- /dev/null +++ b/packages/playwright-test/src/runner/watchMode.ts @@ -0,0 +1,436 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import readline from 'readline'; +import { createGuid, ManualPromise } from 'playwright-core/lib/utils'; +import type { FullConfigInternal, FullProjectInternal } from '../common/types'; +import { Multiplexer } from '../reporters/multiplexer'; +import { createFileMatcher, createFileMatcherFromArguments } from '../util'; +import type { Matcher } from '../util'; +import { createTaskRunnerForWatch, createTaskRunnerForWatchSetup } from './tasks'; +import type { TaskRunnerState } from './tasks'; +import { buildProjectsClosure, filterProjects } from './projectUtils'; +import { clearCompilationCache, collectAffectedTestFiles } from '../common/compilationCache'; +import type { FullResult } from 'packages/playwright-test/reporter'; +import { chokidar } from '../utilsBundle'; +import type { FSWatcher as CFSWatcher } from 'chokidar'; +import { createReporter } from './reporters'; +import { colors } from 'playwright-core/lib/utilsBundle'; +import { enquirer } from '../utilsBundle'; +import { separator } from '../reporters/base'; +import { PlaywrightServer } from 'playwright-core/lib/remote/playwrightServer'; +import ListReporter from '../reporters/list'; + +class FSWatcher { + private _dirtyTestFiles = new Map>(); + private _notifyDirtyFiles: (() => void) | undefined; + private _watcher: CFSWatcher | undefined; + private _timer: NodeJS.Timeout | undefined; + + async update(config: FullConfigInternal) { + const commandLineFileMatcher = config._internal.cliArgs.length ? createFileMatcherFromArguments(config._internal.cliArgs) : () => true; + const projects = filterProjects(config.projects, config._internal.cliProjectFilter); + const projectClosure = buildProjectsClosure(projects); + const projectFilters = new Map(); + for (const project of projectClosure) { + const testMatch = createFileMatcher(project.testMatch); + const testIgnore = createFileMatcher(project.testIgnore); + projectFilters.set(project, file => { + if (!file.startsWith(project.testDir) || !testMatch(file) || testIgnore(file)) + return false; + return project._internal.type === 'dependency' || commandLineFileMatcher(file); + }); + } + + if (this._timer) + clearTimeout(this._timer); + if (this._watcher) + await this._watcher.close(); + + this._watcher = chokidar.watch(projectClosure.map(p => p.testDir), { ignoreInitial: true }).on('all', async (event, file) => { + if (event !== 'add' && event !== 'change') + return; + + const testFiles = new Set(); + collectAffectedTestFiles(file, testFiles); + const testFileArray = [...testFiles]; + + let hasMatches = false; + for (const [project, filter] of projectFilters) { + const filteredFiles = testFileArray.filter(filter); + if (!filteredFiles.length) + continue; + let set = this._dirtyTestFiles.get(project); + if (!set) { + set = new Set(); + this._dirtyTestFiles.set(project, set); + } + filteredFiles.map(f => set!.add(f)); + hasMatches = true; + } + + if (!hasMatches) + return; + + if (this._timer) + clearTimeout(this._timer); + this._timer = setTimeout(() => { + this._notifyDirtyFiles?.(); + }, 250); + }); + + } + + async onDirtyTestFiles(): Promise { + if (this._dirtyTestFiles.size) + return; + await new Promise(f => this._notifyDirtyFiles = f); + } + + takeDirtyTestFiles(): Map> { + const result = this._dirtyTestFiles; + this._dirtyTestFiles = new Map(); + return result; + } +} + +export async function runWatchModeLoop(config: FullConfigInternal): Promise { + // Reset the settings that don't apply to watch. + config._internal.passWithNoTests = true; + for (const p of config.projects) + p.retries = 0; + + // Perform global setup. + const reporter = await createReporter(config, 'watch'); + const context: TaskRunnerState = { + config, + reporter, + phases: [], + }; + const taskRunner = createTaskRunnerForWatchSetup(config, reporter); + reporter.onConfigure(config); + const { status, cleanup: globalCleanup } = await taskRunner.runDeferCleanup(context, 0); + if (status !== 'passed') + return await globalCleanup(); + + // Prepare projects that will be watched, set up watcher. + const failedTestIdCollector = new Set(); + const originalWorkers = config.workers; + const fsWatcher = new FSWatcher(); + await fsWatcher.update(config); + + let lastRun: { type: 'changed' | 'regular' | 'failed', failedTestIds?: Set, dirtyTestFiles?: Map> } = { type: 'regular' }; + let result: FullResult['status'] = 'passed'; + + // Enter the watch loop. + await runTests(config, failedTestIdCollector); + + while (true) { + printPrompt(); + const readCommandPromise = readCommand(); + await Promise.race([ + fsWatcher.onDirtyTestFiles(), + readCommandPromise, + ]); + if (!readCommandPromise.isDone()) + readCommandPromise.resolve('changed'); + + const command = await readCommandPromise; + + if (command === 'changed') { + const dirtyTestFiles = fsWatcher.takeDirtyTestFiles(); + // Resolve files that depend on the changed files. + await runChangedTests(config, failedTestIdCollector, dirtyTestFiles); + lastRun = { type: 'changed', dirtyTestFiles }; + continue; + } + + if (command === 'run') { + // All means reset filters. + await runTests(config, failedTestIdCollector); + lastRun = { type: 'regular' }; + continue; + } + + if (command === 'project') { + const { projectNames } = await enquirer.prompt<{ projectNames: string[] }>({ + type: 'multiselect', + name: 'projectNames', + message: 'Select projects', + choices: config.projects.map(p => ({ name: p.name })), + }).catch(() => ({ projectNames: null })); + if (!projectNames) + continue; + config._internal.cliProjectFilter = projectNames.length ? projectNames : undefined; + await fsWatcher.update(config); + await runTests(config, failedTestIdCollector); + lastRun = { type: 'regular' }; + continue; + } + + if (command === 'file') { + const { filePattern } = await enquirer.prompt<{ filePattern: string }>({ + type: 'text', + name: 'filePattern', + message: 'Input filename pattern (regex)', + }).catch(() => ({ filePattern: null })); + if (filePattern === null) + continue; + if (filePattern.trim()) + config._internal.cliArgs = filePattern.split(' '); + else + config._internal.cliArgs = []; + await fsWatcher.update(config); + await runTests(config, failedTestIdCollector); + lastRun = { type: 'regular' }; + continue; + } + + if (command === 'grep') { + const { testPattern } = await enquirer.prompt<{ testPattern: string }>({ + type: 'text', + name: 'testPattern', + message: 'Input test name pattern (regex)', + }).catch(() => ({ testPattern: null })); + if (testPattern === null) + continue; + if (testPattern.trim()) + config._internal.cliGrep = testPattern; + else + config._internal.cliGrep = undefined; + await fsWatcher.update(config); + await runTests(config, failedTestIdCollector); + lastRun = { type: 'regular' }; + continue; + } + + if (command === 'failed') { + config._internal.testIdMatcher = id => failedTestIdCollector.has(id); + const failedTestIds = new Set(failedTestIdCollector); + await runTests(config, failedTestIdCollector, { title: 'running failed tests' }); + config._internal.testIdMatcher = undefined; + lastRun = { type: 'failed', failedTestIds }; + continue; + } + + if (command === 'repeat') { + if (lastRun.type === 'regular') { + await runTests(config, failedTestIdCollector, { title: 're-running tests' }); + continue; + } else if (lastRun.type === 'changed') { + await runChangedTests(config, failedTestIdCollector, lastRun.dirtyTestFiles!, 're-running tests'); + } else if (lastRun.type === 'failed') { + config._internal.testIdMatcher = id => lastRun.failedTestIds!.has(id); + await runTests(config, failedTestIdCollector, { title: 're-running tests' }); + config._internal.testIdMatcher = undefined; + } + continue; + } + + if (command === 'toggle-show-browser') { + await toggleShowBrowser(config, originalWorkers); + continue; + } + + if (command === 'exit') + break; + + if (command === 'interrupted') { + result = 'interrupted'; + break; + } + } + + return result === 'passed' ? await globalCleanup() : result; +} + +async function runChangedTests(config: FullConfigInternal, failedTestIdCollector: Set, filesByProject: Map>, title?: string) { + const testFiles = new Set(); + for (const files of filesByProject.values()) + files.forEach(f => testFiles.add(f)); + + // Collect all the affected projects, follow project dependencies. + // Prepare to exclude all the projects that do not depend on this file, as if they did not exist. + const projects = filterProjects(config.projects, config._internal.cliProjectFilter); + const projectClosure = buildProjectsClosure(projects); + const affectedProjects = affectedProjectsClosure(projectClosure, [...filesByProject.keys()]); + const affectsAnyDependency = [...affectedProjects].some(p => p._internal.type === 'dependency'); + const projectsToIgnore = new Set(projectClosure.filter(p => !affectedProjects.has(p))); + + // If there are affected dependency projects, do the full run, respect the original CLI. + // if there are no affected dependency projects, intersect CLI with dirty files + const additionalFileMatcher = affectsAnyDependency ? () => true : (file: string) => testFiles.has(file); + await runTests(config, failedTestIdCollector, { projectsToIgnore, additionalFileMatcher, title: title || 'files changed' }); +} + +async function runTests(config: FullConfigInternal, failedTestIdCollector: Set, options?: { + projectsToIgnore?: Set, + additionalFileMatcher?: Matcher, + title?: string, + }) { + printConfiguration(config, options?.title); + const reporter = new Multiplexer([new ListReporter()]); + const taskRunner = createTaskRunnerForWatch(config, reporter, options?.projectsToIgnore, options?.additionalFileMatcher); + const context: TaskRunnerState = { + config, + reporter, + phases: [], + }; + clearCompilationCache(); + reporter.onConfigure(config); + const taskStatus = await taskRunner.run(context, 0); + let status: FullResult['status'] = 'passed'; + + let hasFailedTests = false; + for (const test of context.rootSuite?.allTests() || []) { + if (test.outcome() === 'unexpected') { + failedTestIdCollector.add(test.id); + hasFailedTests = true; + } else { + failedTestIdCollector.delete(test.id); + } + } + + if (context.phases.find(p => p.dispatcher.hasWorkerErrors()) || hasFailedTests) + status = 'failed'; + if (status === 'passed' && taskStatus !== 'passed') + status = taskStatus; + await reporter.onExit({ status }); +} + +function affectedProjectsClosure(projectClosure: FullProjectInternal[], affected: FullProjectInternal[]): Set { + const result = new Set(affected); + for (let i = 0; i < projectClosure.length; ++i) { + for (const p of projectClosure) { + for (const dep of p._internal.deps) { + if (result.has(dep)) + result.add(p); + } + } + } + return result; +} + +function readCommand(): ManualPromise { + const result = new ManualPromise(); + const rl = readline.createInterface({ input: process.stdin, escapeCodeTimeout: 50 }); + readline.emitKeypressEvents(process.stdin, rl); + if (process.stdin.isTTY) + process.stdin.setRawMode(true); + + const handler = (text: string, key: any) => { + if (text === '\x03' || text === '\x1B' || (key && key.name === 'escape') || (key && key.ctrl && key.name === 'c')) { + result.resolve('interrupted'); + return; + } + if (process.platform !== 'win32' && key && key.ctrl && key.name === 'z') { + process.kill(process.ppid, 'SIGTSTP'); + process.kill(process.pid, 'SIGTSTP'); + } + const name = key?.name; + if (name === 'q') { + result.resolve('exit'); + return; + } + if (name === 'h') { + process.stdout.write(`${separator()} +Run tests + ${colors.bold('enter')} ${colors.dim('run tests')} + ${colors.bold('f')} ${colors.dim('run failed tests')} + ${colors.bold('r')} ${colors.dim('repeat last run')} + ${colors.bold('q')} ${colors.dim('quit')} + +Change settings + ${colors.bold('c')} ${colors.dim('set project')} + ${colors.bold('p')} ${colors.dim('set file filter')} + ${colors.bold('t')} ${colors.dim('set title filter')} + ${colors.bold('s')} ${colors.dim('toggle show & reuse the browser')} +`); + return; + } + + switch (name) { + case 'return': result.resolve('run'); break; + case 'r': result.resolve('repeat'); break; + case 'c': result.resolve('project'); break; + case 'p': result.resolve('file'); break; + case 't': result.resolve('grep'); break; + case 'f': result.resolve('failed'); break; + case 's': result.resolve('toggle-show-browser'); break; + } + }; + + process.stdin.on('keypress', handler); + result.finally(() => { + process.stdin.off('keypress', handler); + rl.close(); + if (process.stdin.isTTY) + process.stdin.setRawMode(false); + }); + return result; +} + +let showBrowserServer: PlaywrightServer | undefined; +let seq = 0; + +function printConfiguration(config: FullConfigInternal, title?: string) { + const tokens: string[] = []; + tokens.push('npx playwright test'); + tokens.push(...(config._internal.cliProjectFilter || [])?.map(p => colors.blue(`--project ${p}`))); + if (config._internal.cliGrep) + tokens.push(colors.red(`--grep ${config._internal.cliGrep}`)); + if (config._internal.cliArgs) + tokens.push(...config._internal.cliArgs.map(a => colors.bold(a))); + if (title) + tokens.push(colors.dim(`(${title})`)); + if (seq) + tokens.push(colors.dim(`#${seq}`)); + ++seq; + const lines: string[] = []; + const sep = separator(); + lines.push('\x1Bc' + sep); + lines.push(`${tokens.join(' ')}`); + lines.push(`${colors.dim('Show & reuse browser:')} ${colors.bold(showBrowserServer ? 'on' : 'off')}`); + process.stdout.write(lines.join('\n')); +} + +function printPrompt() { + const sep = separator(); + process.stdout.write(` +${sep} +${colors.dim('Waiting for file changes. Press')} ${colors.bold('enter')} ${colors.dim('to run tests')}, ${colors.bold('q')} ${colors.dim('to quit or')} ${colors.bold('h')} ${colors.dim('for more options.')} +`); +} + +async function toggleShowBrowser(config: FullConfigInternal, originalWorkers: number) { + if (!showBrowserServer) { + config.workers = 1; + showBrowserServer = new PlaywrightServer({ path: '/' + createGuid(), maxConnections: 1 }); + const wsEndpoint = await showBrowserServer.listen(); + process.env.PW_TEST_REUSE_CONTEXT = '1'; + process.env.PW_TEST_CONNECT_WS_ENDPOINT = wsEndpoint; + process.stdout.write(`${colors.dim('Show & reuse browser:')} ${colors.bold('on')}\n`); + } else { + config.workers = originalWorkers; + await showBrowserServer?.close(); + showBrowserServer = undefined; + delete process.env.PW_TEST_REUSE_CONTEXT; + delete process.env.PW_TEST_CONNECT_WS_ENDPOINT; + process.stdout.write(`${colors.dim('Show & reuse browser:')} ${colors.bold('off')}\n`); + } +} + +type Command = 'run' | 'failed' | 'repeat' | 'changed' | 'project' | 'file' | 'grep' | 'exit' | 'interrupted' | 'toggle-show-browser'; diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index c115c73be3..80356e3876 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -20,7 +20,7 @@ import * as os from 'os'; import * as path from 'path'; import { rimraf, PNG } from 'playwright-core/lib/utilsBundle'; import { promisify } from 'util'; -import type { CommonFixtures, CommonWorkerFixtures } from '../config/commonFixtures'; +import type { CommonFixtures, CommonWorkerFixtures, TestChildProcess } from '../config/commonFixtures'; import { commonFixtures } from '../config/commonFixtures'; import type { ServerFixtures, ServerWorkerOptions } from '../config/serverFixtures'; import { serverFixtures } from '../config/serverFixtures'; @@ -155,6 +155,22 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b }; } +function watchPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, env: NodeJS.ProcessEnv, options: RunOptions): TestChildProcess { + const args = ['test', '--workers=2']; + if (options.additionalArgs) + args.push(...options.additionalArgs); + const cwd = options.cwd ? path.resolve(baseDir, options.cwd) : baseDir; + + const command = ['node', cliEntrypoint]; + command.push(...args); + const testProcess = childProcess({ + command, + env: cleanEnv({ PWTEST_WATCH: '1', ...env }), + cwd, + }); + return testProcess; +} + async function runPlaywrightCommand(childProcess: CommonFixtures['childProcess'], cwd: string, commandWithArguments: string[], env: NodeJS.ProcessEnv, sendSIGINTAfter?: number): Promise { const command = ['node', cliEntrypoint]; command.push(...commandWithArguments); @@ -209,6 +225,7 @@ type Fixtures = { writeFiles: (files: Files) => Promise; deleteFile: (file: string) => Promise; runInlineTest: (files: Files, params?: Params, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise; + runWatchTest: (files: Files, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise; runTSC: (files: Files) => Promise; nodeVersion: { major: number, minor: number, patch: number }; }; @@ -237,6 +254,18 @@ export const test = base await removeFolderAsync(cacheDir); }, + runWatchTest: async ({ childProcess }, use, testInfo: TestInfo) => { + const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-')); + let testProcess: TestChildProcess | undefined; + await use(async (files: Files, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => { + const baseDir = await writeFiles(testInfo, files, true); + testProcess = watchPlaywrightTest(childProcess, baseDir, { ...env, PWTEST_CACHE_DIR: cacheDir }, options); + return testProcess; + }); + await testProcess?.kill(); + await removeFolderAsync(cacheDir); + }, + runTSC: async ({ childProcess }, use, testInfo) => { await use(async files => { const baseDir = await writeFiles(testInfo, { 'tsconfig.json': JSON.stringify(TSCONFIG), ...files }, true); diff --git a/tests/playwright-test/watch.spec.ts b/tests/playwright-test/watch.spec.ts index 40908892d7..985774fc70 100644 --- a/tests/playwright-test/watch.spec.ts +++ b/tests/playwright-test/watch.spec.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import path from 'path'; import { test, expect } from './playwright-test-fixtures'; test.describe.configure({ mode: 'parallel' }); @@ -95,3 +96,606 @@ test('should print dependencies in ESM mode', async ({ runInlineTest, nodeVersio 'b.test.ts': ['helperA.ts', 'helperB.ts'], }); }); + +test('should perform initial run', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should quit on Q', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({}, {}); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.write('q'); + await testProcess!.exited; +}); + +test('should print help on H', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({}, {}); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.write('h'); + await testProcess.waitForOutput('to quit'); +}); + +test('should run tests on Enter', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('\r\n'); + await testProcess.waitForOutput('npx playwright test #1'); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should run tests on R', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('r'); + await testProcess.waitForOutput('npx playwright test (re-running tests) #1'); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should run failed tests on F', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'c.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('c.test.ts:3:11 › fails'); + await testProcess.waitForOutput('Error: expect(received).toBe(expected)'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('f'); + await testProcess.waitForOutput('npx playwright test (running failed tests) #1'); + await testProcess.waitForOutput('c.test.ts:3:11 › fails'); + expect(testProcess.output).not.toContain('a.test.ts:3:11'); +}); + +test('should respect file filter P', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('p'); + await testProcess.waitForOutput('Input filename pattern (regex)'); + testProcess.write('b.test\r\n'); + await testProcess.waitForOutput('npx playwright test b.test #1'); + await testProcess.waitForOutput('b.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('a.test.ts:3:11'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should respect project filter C', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({ + 'playwright.config.ts': ` + import { defineConfig } from '@playwright/test'; + export default defineConfig({ projects: [{name: 'foo'}, {name: 'bar'}] }); + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}); + await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('[bar] › a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('c'); + await testProcess.waitForOutput('Select projects'); + await testProcess.waitForOutput('foo'); + await testProcess.waitForOutput('bar'); + testProcess.write(' '); + testProcess.write('\r\n'); + await testProcess.waitForOutput('npx playwright test --project foo #1'); + await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('[bar] › a.test.ts:3:11 › passes'); +}); + +test('should respect file filter P and split files', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('p'); + await testProcess.waitForOutput('Input filename pattern (regex)'); + testProcess.write('a.test b.test\r\n'); + await testProcess.waitForOutput('npx playwright test a.test b.test #1'); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should respect title filter T', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('title 1', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('title 2', () => {}); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › title 1'); + await testProcess.waitForOutput('b.test.ts:3:11 › title 2'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('t'); + await testProcess.waitForOutput('Input test name pattern (regex)'); + testProcess.write('title 2\r\n'); + await testProcess.waitForOutput('npx playwright test --grep title 2 #1'); + await testProcess.waitForOutput('b.test.ts:3:11 › title 2'); + expect(testProcess.output).not.toContain('a.test.ts:3:11'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should re-run failed tests on F > R', async ({ runWatchTest }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'c.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('c.test.ts:3:11 › fails'); + await testProcess.waitForOutput('Error: expect(received).toBe(expected)'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('f'); + await testProcess.waitForOutput('npx playwright test (running failed tests) #1'); + await testProcess.waitForOutput('c.test.ts:3:11 › fails'); + expect(testProcess.output).not.toContain('a.test.ts:3:11'); + testProcess.clearOutput(); + testProcess.write('r'); + await testProcess.waitForOutput('npx playwright test (re-running tests) #2'); + await testProcess.waitForOutput('c.test.ts:3:11 › fails'); + expect(testProcess.output).not.toContain('a.test.ts:3:11'); +}); + +test('should run on changed files', async ({ runWatchTest, writeFiles }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'c.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('c.test.ts:3:11 › fails'); + await testProcess.waitForOutput('Error: expect(received).toBe(expected)'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + await writeFiles({ + 'c.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }); + await testProcess.waitForOutput('c.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('a.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should run on changed deps', async ({ runWatchTest, writeFiles }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import './helper'; + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'helper.ts': ` + console.log('old helper'); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('b.test.ts:4:11 › passes'); + await testProcess.waitForOutput('old helper'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + await writeFiles({ + 'helper.ts': ` + console.log('new helper'); + `, + }); + await testProcess.waitForOutput('b.test.ts:4:11 › passes'); + expect(testProcess.output).not.toContain('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('new helper'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should run on changed deps in ESM', async ({ runWatchTest, writeFiles, nodeVersion }) => { + test.skip(nodeVersion.major < 16); + const testProcess = await runWatchTest({ + 'playwright.config.ts': `export default {};`, + 'package.json': `{ "type": "module" }`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import './helper.js'; + import { test } from '@playwright/test'; + test('passes', () => {}); + `, + 'helper.ts': ` + console.log('old helper'); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:7 › passes'); + await testProcess.waitForOutput('b.test.ts:4:7 › passes'); + await testProcess.waitForOutput('old helper'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + await writeFiles({ + 'helper.ts': ` + console.log('new helper'); + `, + }); + await testProcess.waitForOutput('b.test.ts:4:7 › passes'); + expect(testProcess.output).not.toContain('a.test.ts:3:7 › passes'); + await testProcess.waitForOutput('new helper'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should re-run changed files on R', async ({ runWatchTest, writeFiles }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'c.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('c.test.ts:3:11 › fails'); + await testProcess.waitForOutput('Error: expect(received).toBe(expected)'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + await writeFiles({ + 'c.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }); + await testProcess.waitForOutput('c.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('a.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('r'); + await testProcess.waitForOutput('c.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('a.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should not trigger on changes to non-tests', async ({ runWatchTest, writeFiles }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('b.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); + + testProcess.clearOutput(); + await writeFiles({ + 'helper.ts': ` + console.log('helper'); + `, + }); + + await new Promise(f => setTimeout(f, 1000)); + expect(testProcess.output).not.toContain('Waiting for file changes.'); +}); + +test('should only watch selected projects', async ({ runWatchTest, writeFiles }) => { + const testProcess = await runWatchTest({ + 'playwright.config.ts': ` + import { defineConfig } from '@playwright/test'; + export default defineConfig({ projects: [{name: 'foo'}, {name: 'bar'}] }); + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}, { additionalArgs: ['--project=foo'] }); + await testProcess.waitForOutput('npx playwright test --project foo'); + await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('[bar]'); + await testProcess.waitForOutput('Waiting for file changes.'); + + testProcess.clearOutput(); + await writeFiles({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }); + + await testProcess.waitForOutput('npx playwright test --project foo'); + await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes'); + await testProcess.waitForOutput('Waiting for file changes.'); + expect(testProcess.output).not.toContain('[bar]'); +}); + +test('should watch filtered files', async ({ runWatchTest, writeFiles }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}, { additionalArgs: ['a.test.ts'] }); + await testProcess.waitForOutput('npx playwright test a.test.ts'); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('b.test'); + await testProcess.waitForOutput('Waiting for file changes.'); + + testProcess.clearOutput(); + await writeFiles({ + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }); + + await new Promise(f => setTimeout(f, 1000)); + expect(testProcess.output).not.toContain('Waiting for file changes.'); +}); + +test('should not watch unfiltered files', async ({ runWatchTest, writeFiles }) => { + const testProcess = await runWatchTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }, {}, { additionalArgs: ['a.test.ts'] }); + await testProcess.waitForOutput('npx playwright test a.test.ts'); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('b.test'); + await testProcess.waitForOutput('Waiting for file changes.'); + + testProcess.clearOutput(); + await writeFiles({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', () => {}); + `, + }); + + testProcess.clearOutput(); + await testProcess.waitForOutput('npx playwright test a.test.ts (files changed)'); + await testProcess.waitForOutput('a.test.ts:3:11 › passes'); + expect(testProcess.output).not.toContain('b.test'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should run CT on changed deps', async ({ runWatchTest, writeFiles }) => { + const testProcess = await runWatchTest({ + 'playwright.config.ts': ` + import { defineConfig } from '@playwright/experimental-ct-react'; + export default defineConfig({ projects: [{name: 'default'}] }); + `, + 'playwright/index.html': ``, + 'playwright/index.ts': ``, + 'src/button.tsx': ` + export const Button = () => ; + `, + 'src/button.spec.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + import { Button } from './button'; + test('pass', async ({ mount }) => { + const component = await mount(); + await expect(component).toHaveText('Button', { timeout: 1000 }); + }); + `, + 'src/link.spec.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + test('pass', async ({ mount }) => { + const component = await mount(hello); + await expect(component).toHaveText('hello'); + }); + `, + }, {}); + await testProcess.waitForOutput('button.spec.tsx:4:11 › pass'); + await testProcess.waitForOutput('link.spec.tsx:3:11 › pass'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + await writeFiles({ + 'src/button.tsx': ` + export const Button = () => ; + `, + }); + + await testProcess.waitForOutput(`src${path.sep}button.spec.tsx:4:11 › pass`); + expect(testProcess.output).not.toContain(`src${path.sep}link.spec.tsx`); + await testProcess.waitForOutput('Error: expect(received).toHaveText(expected)'); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should run CT on indirect deps change', async ({ runWatchTest, writeFiles }) => { + const testProcess = await runWatchTest({ + 'playwright.config.ts': ` + import { defineConfig } from '@playwright/experimental-ct-react'; + export default defineConfig({ projects: [{name: 'default'}] }); + `, + 'playwright/index.html': ``, + 'playwright/index.ts': ``, + 'src/button.css': ` + button { color: red; } + `, + 'src/button.tsx': ` + import './button.css'; + export const Button = () => ; + `, + 'src/button.spec.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + import { Button } from './button'; + test('pass', async ({ mount }) => { + const component = await mount(); + await expect(component).toHaveText('Button', { timeout: 1000 }); + }); + `, + 'src/link.spec.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + test('pass', async ({ mount }) => { + const component = await mount(hello); + await expect(component).toHaveText('hello'); + }); + `, + }, {}); + await testProcess.waitForOutput('button.spec.tsx:4:11 › pass'); + await testProcess.waitForOutput('link.spec.tsx:3:11 › pass'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + await writeFiles({ + 'src/button.css': ` + button { color: blue; } + `, + }); + + await testProcess.waitForOutput(`src${path.sep}button.spec.tsx:4:11 › pass`); + expect(testProcess.output).not.toContain(`src${path.sep}link.spec.tsx`); + await testProcess.waitForOutput('Waiting for file changes.'); +}); + +test('should run CT on indirect deps change ESM mode', async ({ runWatchTest, writeFiles, nodeVersion }) => { + test.skip(nodeVersion.major < 16); + const testProcess = await runWatchTest({ + 'playwright.config.ts': ` + import { defineConfig } from '@playwright/experimental-ct-react'; + export default defineConfig({ projects: [{name: 'default'}] }); + `, + 'package.json': `{ "type": "module" }`, + 'playwright/index.html': ``, + 'playwright/index.ts': ``, + 'src/button.css': ` + button { color: red; } + `, + 'src/button.tsx': ` + import './button.css'; + export const Button = () => ; + `, + 'src/button.spec.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + import { Button } from './button.jsx'; + test('pass', async ({ mount }) => { + const component = await mount(); + await expect(component).toHaveText('Button', { timeout: 1000 }); + }); + `, + 'src/link.spec.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + test('pass', async ({ mount }) => { + const component = await mount(hello); + await expect(component).toHaveText('hello'); + }); + `, + }, {}); + await testProcess.waitForOutput('button.spec.tsx:4:7 › pass'); + await testProcess.waitForOutput('link.spec.tsx:3:7 › pass'); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + await writeFiles({ + 'src/button.css': ` + button { color: blue; } + `, + }); + + await testProcess.waitForOutput(`src${path.sep}button.spec.tsx:4:7 › pass`); + expect(testProcess.output).not.toContain(`src${path.sep}link.spec.tsx`); + await testProcess.waitForOutput('Waiting for file changes.'); +});