test(watch): start adding tests (#20764)

2023-02-09 08:31:02 -08:00 · 2023-02-09 08:31:02 -08:00 · b247bfe153
parent 3be6772fa5
commit b247bfe153
4 changed files with 160 additions and 38 deletions
--- a/packages/playwright-test/src/runner/watchMode.ts
+++ b/packages/playwright-test/src/runner/watchMode.ts
@ -245,14 +245,11 @@ async function runChangedTests(config: FullConfigInternal, failedTestIdCollector
  return await runTests(config, failedTestIdCollector, { projectsToIgnore, additionalFileMatcher, title: title || 'files changed' });
 }

-let seq = 0;
-
 async function runTests(config: FullConfigInternal, failedTestIdCollector: Set<string>, options?: {
    projectsToIgnore?: Set<FullProjectInternal>,
    additionalFileMatcher?: Matcher,
    title?: string,
  }) {
-  ++seq;
  printConfiguration(config, options?.title);
  const reporter = new Multiplexer([new ListReporter()]);
  const taskRunner = createTaskRunnerForWatch(config, reporter, options?.projectsToIgnore, options?.additionalFileMatcher);
@ -356,6 +353,7 @@ Change settings
 }

 let showBrowserServer: PlaywrightServer | undefined;
+let seq = 0;

 function printConfiguration(config: FullConfigInternal, title?: string) {
  const tokens: string[] = [];
@ -369,6 +367,7 @@ function printConfiguration(config: FullConfigInternal, title?: string) {
    tokens.push(colors.dim(`(${title})`));
  if (seq)
    tokens.push(colors.dim(`#${seq}`));
+  ++seq;
  const lines: string[] = [];
  const sep = separator();
  lines.push('\x1Bc' + sep);
--- a/tests/config/commonFixtures.ts
+++ b/tests/config/commonFixtures.ts
@ -18,6 +18,7 @@ import type { Fixtures } from '@playwright/test';
 import type { ChildProcess } from 'child_process';
 import { execSync, spawn } from 'child_process';
 import net from 'net';
+import { stripAnsi } from './utils';

 type TestChildParams = {
  command: string[],
@ -105,9 +106,17 @@ export class TestChildProcess {
  }

  async waitForOutput(substring: string) {
-    while (!this.output.includes(substring))
+    while (!stripAnsi(this.output).includes(substring))
      await new Promise<void>(f => this._outputCallbacks.add(f));
  }
+
+  clearOutput() {
+    this.output = '';
+  }
+
+  write(chars: string) {
+    this.process.stdin.write(chars);
+  }
 }

 export type CommonFixtures = {
--- a/tests/playwright-test/playwright-test-fixtures.ts
+++ b/tests/playwright-test/playwright-test-fixtures.ts
@ -20,7 +20,7 @@ import * as os from 'os';
 import * as path from 'path';
 import { rimraf, PNG } from 'playwright-core/lib/utilsBundle';
 import { promisify } from 'util';
-import type { CommonFixtures } from '../config/commonFixtures';
+import type { CommonFixtures, CommonWorkerFixtures, TestChildProcess } from '../config/commonFixtures';
 import { commonFixtures } from '../config/commonFixtures';
 import type { ServerFixtures, ServerWorkerOptions } from '../config/serverFixtures';
 import { serverFixtures } from '../config/serverFixtures';
@ -56,7 +56,6 @@ type TSCResult = {

 type Files = { [key: string]: string | Buffer };
 type Params = { [key: string]: string | number | boolean | string[] };
-type Env = { [key: string]: string | number | boolean | undefined };

 async function writeFiles(testInfo: TestInfo, files: Files) {
  const baseDir = testInfo.outputPath();
@ -110,7 +109,7 @@ async function writeFiles(testInfo: TestInfo, files: Files) {

 const cliEntrypoint = path.join(__dirname, '../../packages/playwright-core/cli.js');

-async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: Env, options: RunOptions): Promise<RunResult> {
+async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: NodeJS.ProcessEnv, options: RunOptions): Promise<RunResult> {
  const paramList: string[] = [];
  for (const key of Object.keys(params)) {
    for (const value of Array.isArray(params[key]) ? params[key] : [params[key]]) {
@ -191,36 +190,34 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
  };
 }

-async function runPlaywrightCommand(childProcess: CommonFixtures['childProcess'], cwd: string, commandWithArguments: string[], env: Env, sendSIGINTAfter?: number): Promise<CliRunResult> {
+function watchPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, env: NodeJS.ProcessEnv, options: RunOptions): TestChildProcess {
+  const paramList: string[] = [];
+  const outputDir = path.join(baseDir, 'test-results');
+  const args = ['test'];
+  args.push('--output=' + outputDir);
+  args.push('--watch');
+  args.push('--workers=2', ...paramList);
+  if (options.additionalArgs)
+    args.push(...options.additionalArgs);
+  const cwd = options.cwd ? path.resolve(baseDir, options.cwd) : baseDir;
+
+  const command = ['node', cliEntrypoint];
+  command.push(...args);
+  const testProcess = childProcess({
+    command,
+    env: cleanEnv(env),
+    cwd,
+  });
+  return testProcess;
+}
+
+async function runPlaywrightCommand(childProcess: CommonFixtures['childProcess'], cwd: string, commandWithArguments: string[], env: NodeJS.ProcessEnv, sendSIGINTAfter?: number): Promise<CliRunResult> {
  const command = ['node', cliEntrypoint];
  command.push(...commandWithArguments);
  const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-'));
  const testProcess = childProcess({
    command,
-    env: {
-      ...process.env,
-      PWTEST_CACHE_DIR: cacheDir,
-      // BEGIN: Reserved CI
-      CI: undefined,
-      BUILD_URL: undefined,
-      CI_COMMIT_SHA: undefined,
-      CI_JOB_URL: undefined,
-      CI_PROJECT_URL: undefined,
-      GITHUB_REPOSITORY: undefined,
-      GITHUB_RUN_ID: undefined,
-      GITHUB_SERVER_URL: undefined,
-      GITHUB_SHA: undefined,
-      // END: Reserved CI
-      PW_TEST_HTML_REPORT_OPEN: undefined,
-      PW_TEST_REPORTER: undefined,
-      PW_TEST_REPORTER_WS_ENDPOINT: undefined,
-      PW_TEST_SOURCE_TRANSFORM: undefined,
-      PW_TEST_SOURCE_TRANSFORM_SCOPE: undefined,
-      TEST_WORKER_INDEX: undefined,
-      TEST_PARLLEL_INDEX: undefined,
-      NODE_OPTIONS: undefined,
-      ...env,
-    },
+    env: cleanEnv(env),
    cwd,
  });
  let didSendSigint = false;
@ -232,10 +229,34 @@ async function runPlaywrightCommand(childProcess: CommonFixtures['childProcess']
  };
  const { exitCode } = await testProcess.exited;
  await removeFolderAsync(cacheDir);
-
  return { exitCode, output: testProcess.output.toString() };
 }

+function cleanEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
+  return {
+    ...process.env,
+    // BEGIN: Reserved CI
+    CI: undefined,
+    BUILD_URL: undefined,
+    CI_COMMIT_SHA: undefined,
+    CI_JOB_URL: undefined,
+    CI_PROJECT_URL: undefined,
+    GITHUB_REPOSITORY: undefined,
+    GITHUB_RUN_ID: undefined,
+    GITHUB_SERVER_URL: undefined,
+    GITHUB_SHA: undefined,
+    // END: Reserved CI
+    PW_TEST_HTML_REPORT_OPEN: undefined,
+    PW_TEST_REPORTER: undefined,
+    PW_TEST_REPORTER_WS_ENDPOINT: undefined,
+    PW_TEST_SOURCE_TRANSFORM: undefined,
+    PW_TEST_SOURCE_TRANSFORM_SCOPE: undefined,
+    TEST_WORKER_INDEX: undefined,
+    TEST_PARLLEL_INDEX: undefined,
+    NODE_OPTIONS: undefined,
+    ...env,
+  };
+}

 type RunOptions = {
  sendSIGINTAfter?: number;
@ -246,15 +267,16 @@ type RunOptions = {
 };
 type Fixtures = {
  writeFiles: (files: Files) => Promise<string>;
-  runInlineTest: (files: Files, params?: Params, env?: Env, options?: RunOptions, beforeRunPlaywrightTest?: ({ baseDir }: { baseDir: string }) => Promise<void>) => Promise<RunResult>;
+  runInlineTest: (files: Files, params?: Params, env?: NodeJS.ProcessEnv, options?: RunOptions, beforeRunPlaywrightTest?: ({ baseDir }: { baseDir: string }) => Promise<void>) => Promise<RunResult>;
+  runWatchTest: (files: Files, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<TestChildProcess>;
  runTSC: (files: Files) => Promise<TSCResult>;
  nodeVersion: { major: number, minor: number, patch: number };
-  runGroups: (files: Files, params?: Params, env?: Env, options?: RunOptions) => Promise<{ timeline: { titlePath: string[], event: 'begin' | 'end' }[] } & RunResult>;
+  runGroups: (files: Files, params?: Params, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<{ timeline: { titlePath: string[], event: 'begin' | 'end' }[] } & RunResult>;
  runCommand: (files: Files, args: string[]) => Promise<CliRunResult>;
 };

 export const test = base
-    .extend<CommonFixtures>(commonFixtures)
+    .extend<CommonFixtures, CommonWorkerFixtures>(commonFixtures)
    .extend<ServerFixtures, ServerWorkerOptions>(serverFixtures)
    .extend<Fixtures>({
      writeFiles: async ({}, use, testInfo) => {
@ -262,12 +284,26 @@ export const test = base
      },

      runInlineTest: async ({ childProcess }, use, testInfo: TestInfo) => {
-        await use(async (files: Files, params: Params = {}, env: Env = {}, options: RunOptions = {}, beforeRunPlaywrightTest?: ({ baseDir: string }) => Promise<void>) => {
+        const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-'));
+        await use(async (files: Files, params: Params = {}, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}, beforeRunPlaywrightTest?: ({ baseDir }: { baseDir: string }) => Promise<void>) => {
          const baseDir = await writeFiles(testInfo, files);
          if (beforeRunPlaywrightTest)
            await beforeRunPlaywrightTest({ baseDir });
-          return await runPlaywrightTest(childProcess, baseDir, params, env, options);
+          return await runPlaywrightTest(childProcess, baseDir, params, { ...env, PWTEST_CACHE_DIR: cacheDir }, options);
        });
+        await removeFolderAsync(cacheDir);
+      },
+
+      runWatchTest: async ({ childProcess }, use, testInfo: TestInfo) => {
+        const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-'));
+        let testProcess: TestChildProcess | undefined;
+        await use(async (files: Files, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => {
+          const baseDir = await writeFiles(testInfo, files);
+          testProcess = watchPlaywrightTest(childProcess, baseDir, { ...env, PWTEST_CACHE_DIR: cacheDir }, options);
+          return testProcess;
+        });
+        await testProcess!.close();
+        await removeFolderAsync(cacheDir);
      },

      runCommand: async ({ childProcess }, use, testInfo: TestInfo) => {
--- a/tests/playwright-test/watch.spec.ts
+++ b/tests/playwright-test/watch.spec.ts
@ -89,3 +89,81 @@ test('should print dependencies in ESM mode', async ({ runInlineTest, nodeVersio
    'b.test.ts': ['helperA.ts', 'helperB.ts'],
  });
 });
+
+test('should perform initial run', async ({ runWatchTest }) => {
+  const testProcess = await runWatchTest({
+    'a.test.ts': `
+      pwt.test('passes', () => {});
+    `,
+  }, {});
+  await testProcess.waitForOutput('a.test.ts:5:11 › passes');
+  await testProcess.waitForOutput('Waiting for file changes.');
+});
+
+test('should quit on Q', async ({ runWatchTest }) => {
+  const testProcess = await runWatchTest({}, {});
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.write('q');
+  await testProcess!.exited;
+});
+
+test('should print help on H', async ({ runWatchTest }) => {
+  const testProcess = await runWatchTest({}, {});
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.write('h');
+  await testProcess.waitForOutput('to quit');
+});
+
+test('should run tests on Enter', async ({ runWatchTest }) => {
+  const testProcess = await runWatchTest({
+    'a.test.ts': `
+      pwt.test('passes', () => {});
+    `,
+  }, {});
+  await testProcess.waitForOutput('a.test.ts:5:11 › passes');
+  await testProcess.waitForOutput('Waiting for file changes.');
+  await testProcess.clearOutput();
+  testProcess.write('\r\n');
+  await testProcess.waitForOutput('npx playwright test #1');
+  await testProcess.waitForOutput('a.test.ts:5:11 › passes');
+  await testProcess.waitForOutput('Waiting for file changes.');
+});
+
+test('should run tests on R', async ({ runWatchTest }) => {
+  const testProcess = await runWatchTest({
+    'a.test.ts': `
+      pwt.test('passes', () => {});
+    `,
+  }, {});
+  await testProcess.waitForOutput('a.test.ts:5:11 › passes');
+  await testProcess.waitForOutput('Waiting for file changes.');
+  await testProcess.clearOutput();
+  testProcess.write('r');
+  await testProcess.waitForOutput('npx playwright test (re-running tests) #1');
+  await testProcess.waitForOutput('a.test.ts:5:11 › passes');
+  await testProcess.waitForOutput('Waiting for file changes.');
+});
+
+test('should re-run failed tests on F', async ({ runWatchTest }) => {
+  const testProcess = await runWatchTest({
+    'a.test.ts': `
+      pwt.test('passes', () => {});
+    `,
+    'b.test.ts': `
+      pwt.test('passes', () => {});
+    `,
+    'c.test.ts': `
+      pwt.test('fails', () => { expect(1).toBe(2); });
+    `,
+  }, {});
+  await testProcess.waitForOutput('a.test.ts:5:11 › passes');
+  await testProcess.waitForOutput('b.test.ts:5:11 › passes');
+  await testProcess.waitForOutput('c.test.ts:5:11 › fails');
+  await testProcess.waitForOutput('Error: expect(received).toBe(expected)');
+  await testProcess.waitForOutput('Waiting for file changes.');
+  await testProcess.clearOutput();
+  testProcess.write('f');
+  await testProcess.waitForOutput('npx playwright test (running failed tests) #1');
+  await testProcess.waitForOutput('c.test.ts:5:11 › fails');
+  expect(testProcess.output).not.toContain('a.test.ts:5:11');
+});