fix(test runner): do not optimize filtering when sourcemap is present (#21359)

Fixes #21204.

packages/playwright-test/src/runner/loadUtils.ts

@@ -14,13 +14,15 @@
  * limitations under the License.
  */
 
+import fs from 'fs';
 import path from 'path';
+import readline from 'readline';
 import type { Reporter, TestError } from '../../types/testReporter';
 import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost';
 import { Suite } from '../common/test';
 import type { TestCase } from '../common/test';
 import type { FullConfigInternal, FullProjectInternal } from '../common/types';
-import { createFileFiltersFromArguments, createTitleMatcher, errorWithFile, forceRegExp } from '../util';
+import { createFileMatcherFromArguments, createFileFiltersFromArguments, createTitleMatcher, errorWithFile, forceRegExp } from '../util';
 import type { Matcher, TestFileFilter } from '../util';
 import { buildProjectsClosure, collectFilesForProject, filterProjects } from './projectUtils';
 import { requireOrImport } from '../common/transform';
@@ -28,15 +30,23 @@ import { buildFileSuiteForProject, filterByFocusedLine, filterByTestIds, filterO
 import { filterForShard } from './testGroups';
 import { dependenciesForTestFile } from '../common/compilationCache';
 
-export async function loadAllTests(mode: 'out-of-process' | 'in-process', config: FullConfigInternal, projectsToIgnore: Set<FullProjectInternal>, fileMatcher: Matcher, errors: TestError[], shouldFilterOnly: boolean): Promise<Suite> {
+export async function loadAllTests(mode: 'out-of-process' | 'in-process', config: FullConfigInternal, projectsToIgnore: Set<FullProjectInternal>, additionalFileMatcher: Matcher | undefined, errors: TestError[], shouldFilterOnly: boolean): Promise<Suite> {
   const projects = filterProjects(config.projects, config._internal.cliProjectFilter);
 
+  // Interpret cli parameters.
+  const cliFileFilters = createFileFiltersFromArguments(config._internal.cliArgs);
+  const grepMatcher = config._internal.cliGrep ? createTitleMatcher(forceRegExp(config._internal.cliGrep)) : () => true;
+  const grepInvertMatcher = config._internal.cliGrepInvert ? createTitleMatcher(forceRegExp(config._internal.cliGrepInvert)) : () => false;
+  const cliTitleMatcher = (title: string) => !grepInvertMatcher(title) && grepMatcher(title);
+  const cliFileMatcher = config._internal.cliArgs.length ? createFileMatcherFromArguments(config._internal.cliArgs) : null;
+
   let filesToRunByProject = new Map<FullProjectInternal, string[]>();
   let topLevelProjects: FullProjectInternal[];
   let dependencyProjects: FullProjectInternal[];
   // Collect files, categorize top level and dependency projects.
   {
     const fsCache = new Map();
+    const sourceMapCache = new Map();
 
     // First collect all files for the projects in the command line, don't apply any file filters.
     const allFilesForProject = new Map<FullProjectInternal, string[]>();
@@ -49,7 +59,16 @@ export async function loadAllTests(mode: 'out-of-process' | 'in-process', config
 
     // Filter files based on the file filters, eliminate the empty projects.
     for (const [project, files] of allFilesForProject) {
-      const filteredFiles = files.filter(fileMatcher);
+      const matchedFiles = await Promise.all(files.map(async file => {
+        if (additionalFileMatcher && !additionalFileMatcher(file))
+          return;
+        if (cliFileMatcher) {
+          if (!cliFileMatcher(file) && !await isPotentiallyJavaScriptFileWithSourceMap(file, sourceMapCache))
+            return;
+        }
+        return file;
+      }));
+      const filteredFiles = matchedFiles.filter(Boolean) as string[];
       if (filteredFiles.length)
         filesToRunByProject.set(project, filteredFiles);
     }
@@ -113,12 +132,6 @@ export async function loadAllTests(mode: 'out-of-process' | 'in-process', config
   // Create root suites with clones for the projects.
   const rootSuite = new Suite('', 'root');
 
-  // Interpret cli parameters.
-  const cliFileFilters = createFileFiltersFromArguments(config._internal.cliArgs);
-  const grepMatcher = config._internal.cliGrep ? createTitleMatcher(forceRegExp(config._internal.cliGrep)) : () => true;
-  const grepInvertMatcher = config._internal.cliGrepInvert ? createTitleMatcher(forceRegExp(config._internal.cliGrepInvert)) : () => false;
-  const cliTitleMatcher = (title: string) => !grepInvertMatcher(title) && grepMatcher(title);
-
   // First iterate leaf projects to focus only, then add all other projects.
   for (const project of topLevelProjects) {
     const projectSuite = await createProjectSuite(fileSuites, project, { cliFileFilters, cliTitleMatcher, testIdMatcher: config._internal.testIdMatcher }, filesToRunByProject.get(project)!);
@@ -239,3 +252,30 @@ export function loadGlobalHook(config: FullConfigInternal, file: string): Promis
 export function loadReporter(config: FullConfigInternal, file: string): Promise<new (arg?: any) => Reporter> {
   return requireOrImportDefaultFunction(path.resolve(config.rootDir, file), true);
 }
+
+async function isPotentiallyJavaScriptFileWithSourceMap(file: string, cache: Map<string, boolean>): Promise<boolean> {
+  if (!file.endsWith('.js'))
+    return false;
+  if (cache.has(file))
+    return cache.get(file)!;
+
+  try {
+    const stream = fs.createReadStream(file);
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+    let lastLine: string | undefined;
+    rl.on('line', line => {
+      lastLine = line;
+    });
+    await new Promise((fulfill, reject) => {
+      rl.on('close', fulfill);
+      rl.on('error', reject);
+      stream.on('error', reject);
+    });
+    const hasSourceMap = !!lastLine && lastLine.startsWith('//# sourceMappingURL=');
+    cache.set(file, hasSourceMap);
+    return hasSourceMap;
+  } catch (e) {
+    cache.set(file, true);
+    return true;
+  }
+}

packages/playwright-test/src/runner/tasks.ts

@@ -28,7 +28,6 @@ import { TaskRunner } from './taskRunner';
 import type { Suite } from '../common/test';
 import type { FullConfigInternal, FullProjectInternal } from '../common/types';
 import { loadAllTests, loadGlobalHook } from './loadUtils';
-import { createFileMatcherFromArguments } from '../util';
 import type { Matcher } from '../util';
 
 const removeFolderAsync = promisify(rimraf);
@@ -158,9 +157,7 @@ function createRemoveOutputDirsTask(): Task<TaskRunnerState> {
 function createLoadTask(mode: 'out-of-process' | 'in-process', shouldFilterOnly: boolean, projectsToIgnore = new Set<FullProjectInternal>(), additionalFileMatcher?: Matcher): Task<TaskRunnerState> {
   return async (context, errors) => {
     const { config } = context;
-    const cliMatcher = config._internal.cliArgs.length ? createFileMatcherFromArguments(config._internal.cliArgs) : () => true;
-    const fileMatcher = (value: string) => cliMatcher(value) && (additionalFileMatcher ? additionalFileMatcher(value) : true);
-    context.rootSuite = await loadAllTests(mode, config, projectsToIgnore, fileMatcher, errors, shouldFilterOnly);
+    context.rootSuite = await loadAllTests(mode, config, projectsToIgnore, additionalFileMatcher, errors, shouldFilterOnly);
     // Fail when no tests.
     if (!context.rootSuite.allTests().length && !config._internal.passWithNoTests && !config.shard)
       throw new Error(`No tests found`);

tests/playwright-test/runner.spec.ts

@@ -501,3 +501,23 @@ test('should not load tests not matching filter', async ({ runInlineTest }) => {
   expect(result.output).not.toContain('in example.spec.ts');
   expect(result.output).toContain('in a.spec.ts');
 });
+
+test('should filter by sourcemapped file names', async ({ runInlineTest }) => {
+  const fileWithSourceMap = `` +
+`import {test} from '@playwright/test';
+test.describe('Some describe', ()=>{
+  test('Some test', async ()=>{
+    console.log('test')
+  })
+})
+//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImdoZXJraW4uZmVhdHVyZSJdLCJuYW1lcyI6WyJOb25lIl0sIm1hcHBpbmdzIjoiQUFBQUE7QUFBQUE7QUFBQUE7QUFBQUE7QUFBQUE7QUFBQUEiLCJmaWxlIjoiZ2hlcmtpbi5mZWF0dXJlIiwic291cmNlc0NvbnRlbnQiOlsiVGVzdCJdfQ==`;
+
+  const result = await runInlineTest({
+    'playwright.config.js': `export default { projects: [{}, {}] }`,
+    'a.spec.js': fileWithSourceMap,
+  }, {}, {}, { additionalArgs: ['gherkin.feature'] });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+  expect(result.output).not.toContain('a.spec.js');
+  expect(result.output).toContain('gherkin.feature:1');
+});