feat(test runner): --only-changed

packages/playwright/src/common/config.ts

@@ -49,6 +49,7 @@ export class FullConfigInternal {
   cliArgs: string[] = [];
   cliGrep: string | undefined;
   cliGrepInvert: string | undefined;
+  cliOnlyChanged = false;
   cliProjectFilter?: string[];
   cliListOnly = false;
   cliPassWithNoTests?: boolean;

packages/playwright/src/program.ts

@@ -192,6 +192,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
 
   config.cliArgs = args;
   config.cliGrep = opts.grep as string | undefined;
+  config.cliOnlyChanged = !!opts.onlyChanged;
   config.cliGrepInvert = opts.grepInvert as string | undefined;
   config.cliListOnly = !!opts.list;
   config.cliProjectFilter = opts.project || undefined;
@@ -352,6 +353,7 @@ const testOptions: [string, string][] = [
   ['--max-failures <N>', `Stop after the first N failures`],
   ['--no-deps', 'Do not run project dependencies'],
   ['--output <dir>', `Folder for output artifacts (default: "test-results")`],
+  ['--only-changed', `something something docs`],
   ['--pass-with-no-tests', `Makes test run succeed even if no tests were found`],
   ['--project <project-name...>', `Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects)`],
   ['--quiet', `Suppress stdio`],

packages/playwright/src/runner/loadUtils.ts

@@ -36,7 +36,7 @@ export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTest
   const config = testRun.config;
   const fsCache = new Map();
   const sourceMapCache = new Map();
-  const cliFileMatcher = config.cliArgs.length ? createFileMatcherFromArguments(config.cliArgs) : null;
+  const cliFileMatcher = (config.cliArgs.length || config.cliOnlyChanged) ? await createFileMatcherFromArguments(config.cliArgs, config.cliOnlyChanged) : null;
 
   // First collect all files for the projects in the command line, don't apply any file filters.
   const allFilesForProject = new Map<FullProjectInternal, string[]>();
@@ -128,7 +128,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
   // Filter all the projects using grep, testId, file names.
   {
     // Interpret cli parameters.
-    const cliFileFilters = createFileFiltersFromArguments(config.cliArgs);
+    const cliFileFilters = await createFileFiltersFromArguments(config.cliArgs, config.cliOnlyChanged);
     const grepMatcher = config.cliGrep ? createTitleMatcher(forceRegExp(config.cliGrep)) : () => true;
     const grepInvertMatcher = config.cliGrepInvert ? createTitleMatcher(forceRegExp(config.cliGrepInvert)) : () => false;
     const cliTitleMatcher = (title: string) => !grepInvertMatcher(title) && grepMatcher(title);

packages/playwright/src/runner/watchMode.ts

@@ -39,7 +39,7 @@ class FSWatcher {
   private _timer: NodeJS.Timeout | undefined;
 
   async update(config: FullConfigInternal) {
-    const commandLineFileMatcher = config.cliArgs.length ? createFileMatcherFromArguments(config.cliArgs) : () => true;
+    const commandLineFileMatcher = (config.cliArgs.length || config.cliOnlyChanged) ? await createFileMatcherFromArguments(config.cliArgs, config.cliOnlyChanged) : () => true;
     const projects = filterProjects(config.projects, config.cliProjectFilter);
     const projectClosure = buildProjectsClosure(projects);
     const projectFilters = new Map<FullProjectInternal, Matcher>();

packages/playwright/src/util.ts

@@ -19,6 +19,7 @@ import type { StackFrame } from '@protocol/channels';
 import util from 'util';
 import path from 'path';
 import url from 'url';
+import childProcess from 'child_process';
 import { debug, mime, minimatch, parseStackTraceLine } from 'playwright-core/lib/utilsBundle';
 import { formatCallLog } from 'playwright-core/lib/utils';
 import type { TestInfoError } from './../types/test';
@@ -79,7 +80,12 @@ export type TestFileFilter = {
   column: number | null;
 };
 
-export function createFileFiltersFromArguments(args: string[]): TestFileFilter[] {
+export async function createFileFiltersFromArguments(args: string[], onlyChanged: boolean): Promise<TestFileFilter[]> {
+  if (onlyChanged) {
+    const untrackedFiles = childProcess.execSync('git ls-files --others --exclude-standard', { encoding: 'utf-8' }).split('\n').filter(Boolean);
+    args = untrackedFiles;
+  }
+
   return args.map(arg => {
     const match = /^(.*?):(\d+):?(\d+)?$/.exec(arg);
     return {
@@ -90,8 +96,8 @@ export function createFileFiltersFromArguments(args: string[]): TestFileFilter[]
   });
 }
 
-export function createFileMatcherFromArguments(args: string[]): Matcher {
+export async function createFileMatcherFromArguments(args: string[], onlyChanged: boolean): Promise<Matcher> {
-  const filters = createFileFiltersFromArguments(args);
+  const filters = await createFileFiltersFromArguments(args, onlyChanged);
   return createFileMatcher(filters.map(filter => filter.re || filter.exact || ''));
 }
 

tests/playwright-test/only-changed.spec.ts

@@ -0,0 +1,47 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, magicFileCreationSymbol } from './playwright-test-fixtures';
+import { execSync } from 'node:child_process';
+import { writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+test('should filter by file name', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+        import { test, expect } from '@playwright/test';
+        test('fails', () => { expect(1).toBe(2); });
+      `,
+    'b.spec.ts': `
+        import { test, expect } from '@playwright/test';
+        test('fails', () => { expect(1).toBe(2); });
+      `,
+    async [magicFileCreationSymbol](baseDir) {
+      execSync(`git init --initial-branch=main`, { cwd: baseDir });
+      execSync(`git add .`, { cwd: baseDir });
+      execSync(`git commit -m init`, { cwd: baseDir });
+
+      await writeFile(join(baseDir, 'c.spec.ts'), `
+        import { test, expect } from '@playwright/test';
+        test('fails', () => { expect(1).toBe(2); });
+      `);
+    }
+  }, { 'only-changed': true });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('c.spec.ts');
+});

tests/playwright-test/playwright-test-fixtures.ts

@@ -56,7 +56,11 @@ type TSCResult = {
   exitCode: number;
 };
 
-export type Files = { [key: string]: string | Buffer };
+export const magicFileCreationSymbol = Symbol();
+export type Files = {
+  [key: string]: string | Buffer;
+  [magicFileCreationSymbol]?: (baseDir: string) => Promise<void>;
+};
 type Params = { [key: string]: string | number | boolean | string[] };
 
 export async function writeFiles(testInfo: TestInfo, files: Files, initial: boolean) {
@@ -84,6 +88,9 @@ export async function writeFiles(testInfo: TestInfo, files: Files, initial: bool
     await fs.promises.writeFile(fullName, files[name]);
   }));
 
+  if (magicFileCreationSymbol in files)
+    await files[magicFileCreationSymbol](baseDir);
+
   return baseDir;
 }
 
@@ -232,7 +239,7 @@ export function cleanEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
     PWTEST_BOT_NAME: undefined,
     TEST_WORKER_INDEX: undefined,
     TEST_PARALLEL_INDEX: undefined,
-    NODE_OPTIONS: undefined,
+    // NODE_OPTIONS: undefined,
     ...env,
   };
 }