feat(test runner): don't run tests on --watch start (#32583)

Closes https://github.com/microsoft/playwright/issues/32580.

packages/playwright/src/isomorphic/testServerInterface.ts

@@ -28,6 +28,7 @@ export interface TestServerInterface {
     closeOnDisconnect?: boolean,
     interceptStdio?: boolean,
     watchTestDirs?: boolean,
+    populateDependenciesOnList?: boolean,
   }): Promise<void>;
 
   ping(params: {}): Promise<void>;

packages/playwright/src/runner/testServer.ts

@@ -79,6 +79,7 @@ export class TestServerDispatcher implements TestServerInterface {
   private _serializer = require.resolve('./uiModeReporter');
   private _watchTestDirs = false;
   private _closeOnDisconnect = false;
+  private _populateDependenciesOnList = false;
 
   constructor(configLocation: ConfigLocation) {
     this._configLocation = configLocation;
@@ -113,6 +114,7 @@ export class TestServerDispatcher implements TestServerInterface {
     this._closeOnDisconnect = !!params.closeOnDisconnect;
     await this._setInterceptStdio(!!params.interceptStdio);
     this._watchTestDirs = !!params.watchTestDirs;
+    this._populateDependenciesOnList = !!params.populateDependenciesOnList;
   }
 
   async ping() {}
@@ -252,7 +254,7 @@ export class TestServerDispatcher implements TestServerInterface {
     config.cliListOnly = true;
 
     const status = await runTasks(new TestRun(config, reporter), [
-      createLoadTask('out-of-process', { failOnLoadErrors: false, filterOnly: false }),
+      createLoadTask('out-of-process', { failOnLoadErrors: false, filterOnly: false, populateDependencies: this._populateDependenciesOnList }),
       createReportBeginTask(),
     ]);
     return { config, report, reporter, status };

packages/playwright/src/runner/watchMode.ts

@@ -122,7 +122,7 @@ export async function runWatchModeLoop(configLocation: ConfigLocation, initialOp
   });
   testServerConnection.onReport(report => teleSuiteUpdater.processTestReportEvent(report));
 
-  await testServerConnection.initialize({ interceptStdio: false, watchTestDirs: true });
+  await testServerConnection.initialize({ interceptStdio: false, watchTestDirs: true, populateDependenciesOnList: true });
   await testServerConnection.runGlobalSetup({});
 
   const { report } = await testServerConnection.listTests({});
@@ -133,9 +133,6 @@ export async function runWatchModeLoop(configLocation: ConfigLocation, initialOp
   let lastRun: { type: 'changed' | 'regular' | 'failed', failedTestIds?: string[], dirtyTestIds?: string[] } = { type: 'regular' };
   let result: FullResult['status'] = 'passed';
 
-  // Enter the watch loop.
-  await runTests(options, testServerConnection);
-
   while (true) {
     printPrompt();
     const readCommandPromise = readCommand();
@@ -330,7 +327,7 @@ Change settings
 
 let showBrowserServer: PlaywrightServer | undefined;
 let connectWsEndpoint: string | undefined = undefined;
-let seq = 0;
+let seq = 1;
 
 function printConfiguration(options: WatchModeOptions, title?: string) {
   const packageManagerCommand = getPackageManagerExecCommand();
@@ -344,9 +341,7 @@ function printConfiguration(options: WatchModeOptions, title?: string) {
     tokens.push(...options.files.map(a => colors.bold(a)));
   if (title)
     tokens.push(colors.dim(`(${title})`));
-  if (seq)
+  tokens.push(colors.dim(`#${seq++}`));
-    tokens.push(colors.dim(`#${seq}`));
-  ++seq;
   const lines: string[] = [];
   const sep = separator();
   lines.push('\x1Bc' + sep);

tests/playwright-test/watch.spec.ts

@@ -174,15 +174,16 @@ test('should print dependencies in mixed CJS/ESM mode 2', async ({ runInlineTest
   });
 });
 
-test('should perform initial run', async ({ runWatchTest }) => {
+test('should not perform initial run', async ({ runWatchTest }) => {
   const testProcess = await runWatchTest({
     'a.test.ts': `
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
+
+  expect(testProcess.output).not.toContain('a.test.ts');
 });
 
 test('should quit on Q', async ({ runWatchTest }) => {
@@ -206,7 +207,6 @@ test('should run tests on Enter', async ({ runWatchTest }) => {
       test('passes', () => {});
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   testProcess.write('\r\n');
@@ -222,7 +222,6 @@ test('should run tests on R', async ({ runWatchTest }) => {
       test('passes', () => {});
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   testProcess.write('r');
@@ -246,6 +245,10 @@ test('should run failed tests on F', async ({ runWatchTest }) => {
       test('fails', () => { expect(1).toBe(2); });
     `,
   });
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.write('\r\n');
+
+  await testProcess.waitForOutput('npx playwright test #1');
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('c.test.ts:3:11 › fails');
@@ -253,7 +256,7 @@ test('should run failed tests on F', async ({ runWatchTest }) => {
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   testProcess.write('f');
-  await testProcess.waitForOutput('npx playwright test (running failed tests) #1');
+  await testProcess.waitForOutput('npx playwright test (running failed tests) #2');
   await testProcess.waitForOutput('c.test.ts:3:11 › fails');
   expect(testProcess.output).not.toContain('a.test.ts:3:11');
 });
@@ -269,8 +272,6 @@ test('should respect file filter P', async ({ runWatchTest }) => {
       test('passes', () => {});
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › passes');
-  await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   testProcess.write('p');
@@ -294,6 +295,11 @@ test('should respect project filter C', async ({ runWatchTest, writeFiles }) =>
     `,
   };
   const testProcess = await runWatchTest(files, { project: 'foo' });
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.clearOutput();
+  testProcess.write('\r\n');
+
+  await testProcess.waitForOutput('npx playwright test --project foo #1');
   await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
@@ -303,7 +309,7 @@ test('should respect project filter C', async ({ runWatchTest, writeFiles }) =>
   await testProcess.waitForOutput('bar');
   testProcess.write(' ');
   testProcess.write('\r\n');
-  await testProcess.waitForOutput('npx playwright test --project foo #1');
+  await testProcess.waitForOutput('npx playwright test --project foo #2');
   await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes');
   expect(testProcess.output).not.toContain('[bar] › a.test.ts:3:11 › passes');
 
@@ -329,8 +335,6 @@ test('should respect file filter P and split files', async ({ runWatchTest }) =>
       test('passes', () => {});
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › passes');
-  await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   testProcess.write('p');
@@ -353,8 +357,6 @@ test('should respect title filter T', async ({ runWatchTest }) => {
       test('title 2', () => {});
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › title 1');
-  await testProcess.waitForOutput('b.test.ts:3:11 › title 2');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   testProcess.write('t');
@@ -381,6 +383,11 @@ test('should re-run failed tests on F > R', async ({ runWatchTest }) => {
       test('fails', () => { expect(1).toBe(2); });
     `,
   });
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.clearOutput();
+  testProcess.write('\r\n');
+
+  await testProcess.waitForOutput('npx playwright test #1');
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('c.test.ts:3:11 › fails');
@@ -388,12 +395,12 @@ test('should re-run failed tests on F > R', async ({ runWatchTest }) => {
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   testProcess.write('f');
-  await testProcess.waitForOutput('npx playwright test (running failed tests) #1');
+  await testProcess.waitForOutput('npx playwright test (running failed tests) #2');
   await testProcess.waitForOutput('c.test.ts:3:11 › fails');
   expect(testProcess.output).not.toContain('a.test.ts:3:11');
   testProcess.clearOutput();
   testProcess.write('r');
-  await testProcess.waitForOutput('npx playwright test (re-running tests) #2');
+  await testProcess.waitForOutput('npx playwright test (re-running tests) #3');
   await testProcess.waitForOutput('c.test.ts:3:11 › fails');
   expect(testProcess.output).not.toContain('a.test.ts:3:11');
 });
@@ -413,10 +420,6 @@ test('should run on changed files', async ({ runWatchTest, writeFiles }) => {
       test('fails', () => { expect(1).toBe(2); });
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › passes');
-  await testProcess.waitForOutput('b.test.ts:3:11 › passes');
-  await testProcess.waitForOutput('c.test.ts:3:11 › fails');
-  await testProcess.waitForOutput('Error: expect(received).toBe(expected)');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   await writeFiles({
@@ -457,9 +460,6 @@ test('should run on changed deps', async ({ runWatchTest, writeFiles }) => {
       console.log('old helper');
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › passes');
-  await testProcess.waitForOutput('b.test.ts:4:11 › passes');
-  await testProcess.waitForOutput('old helper');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   await writeFiles({
@@ -490,9 +490,6 @@ test('should run on changed deps in ESM', async ({ runWatchTest, writeFiles }) =
       console.log('old helper');
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:7 › passes');
-  await testProcess.waitForOutput('b.test.ts:4:7 › passes');
-  await testProcess.waitForOutput('old helper');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   await writeFiles({
@@ -521,10 +518,6 @@ test('should re-run changed files on R', async ({ runWatchTest, writeFiles }) =>
       test('fails', () => { expect(1).toBe(2); });
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › passes');
-  await testProcess.waitForOutput('b.test.ts:3:11 › passes');
-  await testProcess.waitForOutput('c.test.ts:3:11 › fails');
-  await testProcess.waitForOutput('Error: expect(received).toBe(expected)');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
   await writeFiles({
@@ -556,8 +549,6 @@ test('should not trigger on changes to non-tests', async ({ runWatchTest, writeF
       test('passes', () => {});
     `,
   });
-  await testProcess.waitForOutput('a.test.ts:3:11 › passes');
-  await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
 
   testProcess.clearOutput();
@@ -582,6 +573,10 @@ test('should only watch selected projects', async ({ runWatchTest, writeFiles })
       test('passes', () => {});
     `,
   }, undefined, undefined, { additionalArgs: ['--project=foo'] });
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.clearOutput();
+  testProcess.write('\r\n');
+
   await testProcess.waitForOutput('npx playwright test --project foo');
   await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes');
   expect(testProcess.output).not.toContain('[bar]');
@@ -612,6 +607,10 @@ test('should watch filtered files', async ({ runWatchTest, writeFiles }) => {
       test('passes', () => {});
     `,
   }, undefined, undefined, { additionalArgs: ['a.test.ts'] });
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.clearOutput();
+  testProcess.write('\r\n');
+
   await testProcess.waitForOutput('npx playwright test a.test.ts');
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   expect(testProcess.output).not.toContain('b.test');
@@ -640,6 +639,10 @@ test('should not watch unfiltered files', async ({ runWatchTest, writeFiles }) =
       test('passes', () => {});
     `,
   }, undefined, undefined, { additionalArgs: ['a.test.ts'] });
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.clearOutput();
+  testProcess.write('\r\n');
+
   await testProcess.waitForOutput('npx playwright test a.test.ts');
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   expect(testProcess.output).not.toContain('b.test');
@@ -684,10 +687,7 @@ test('should run CT on changed deps', async ({ runWatchTest, writeFiles }) => {
       });
     `,
   });
-  await testProcess.waitForOutput('button.spec.tsx:4:11 › pass');
-  await testProcess.waitForOutput('link.spec.tsx:3:11 › pass');
   await testProcess.waitForOutput('Waiting for file changes.');
-  testProcess.clearOutput();
   await writeFiles({
     'src/button.tsx': `
       export const Button = () => <button>Button 2</button>;
@@ -732,10 +732,7 @@ test('should run CT on indirect deps change', async ({ runWatchTest, writeFiles
       });
     `,
   });
-  await testProcess.waitForOutput('button.spec.tsx:4:11 › pass');
-  await testProcess.waitForOutput('link.spec.tsx:3:11 › pass');
   await testProcess.waitForOutput('Waiting for file changes.');
-  testProcess.clearOutput();
   await writeFiles({
     'src/button.css': `
       button { color: blue; }
@@ -776,10 +773,7 @@ test('should run CT on indirect deps change ESM mode', async ({ runWatchTest, wr
       });
     `,
   });
-  await testProcess.waitForOutput('button.spec.tsx:4:7 › pass');
-  await testProcess.waitForOutput('link.spec.tsx:3:7 › pass');
   await testProcess.waitForOutput('Waiting for file changes.');
-  testProcess.clearOutput();
   await writeFiles({
     'src/button.css': `
       button { color: blue; }
@@ -809,6 +803,10 @@ test('should run global teardown before exiting', async ({ runWatchTest }) => {
       });
     `,
   });
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.clearOutput();
+  testProcess.write('\r\n');
+
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.write('\x1B');