diff --git a/packages/playwright-test/src/common/configLoader.ts b/packages/playwright-test/src/common/configLoader.ts index ae5dcc24e3..52ea10d7e1 100644 --- a/packages/playwright-test/src/common/configLoader.ts +++ b/packages/playwright-test/src/common/configLoader.ts @@ -146,6 +146,8 @@ export class ConfigLoader { } this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata); this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath)); + + resolveProjectDependencies(this._fullConfig.projects); this._assignUniqueProjectIds(this._fullConfig.projects); } @@ -216,6 +218,8 @@ export class ConfigLoader { testMatch: takeFirst(projectConfig.testMatch, config.testMatch, '**/?(*.)@(spec|test).*'), timeout: takeFirst(projectConfig.timeout, config.timeout, defaultTimeout), use: mergeObjects(config.use, projectConfig.use), + _deps: (projectConfig as any)._deps || [], + _depProjects: [], }; } } @@ -454,6 +458,19 @@ function resolveScript(id: string, rootDir: string) { return require.resolve(id, { paths: [rootDir] }); } +function resolveProjectDependencies(projects: FullProjectInternal[]) { + for (const project of projects) { + for (const dependencyName of project._deps) { + const dependencies = projects.filter(p => p.name === dependencyName); + if (!dependencies.length) + throw new Error(`Project '${project.name}' depends on unknown project '${dependencyName}'`); + if (dependencies.length > 1) + throw new Error(`Project dependencies should have unique names, reading ${dependencyName}`); + project._depProjects.push(...dependencies); + } + } +} + export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs']; export function resolveConfigFile(configFileOrDirectory: string): string | null { diff --git a/packages/playwright-test/src/common/suiteUtils.ts b/packages/playwright-test/src/common/suiteUtils.ts index dd5640ef7a..eb48419e38 100644 --- a/packages/playwright-test/src/common/suiteUtils.ts +++ b/packages/playwright-test/src/common/suiteUtils.ts @@ -16,49 +16,11 @@ import path from 'path'; import { calculateSha1 } from 'playwright-core/lib/utils'; -import type { TestCase } from './test'; -import { Suite } from './test'; +import type { Suite, TestCase } from './test'; import type { FullProjectInternal } from './types'; -import type { Matcher } from '../util'; -import { createTitleMatcher } from '../util'; +import type { TestFileFilter } from '../util'; +import { createFileMatcher } from '../util'; -export async function createRootSuite(preprocessRoot: Suite, testTitleMatcher: Matcher, filesByProject: Map): Promise { - // Generate projects. - const fileSuites = new Map(); - for (const fileSuite of preprocessRoot.suites) - fileSuites.set(fileSuite._requireFile, fileSuite); - - const rootSuite = new Suite('', 'root'); - for (const [project, files] of filesByProject) { - const grepMatcher = createTitleMatcher(project.grep); - const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null; - - const titleMatcher = (test: TestCase) => { - const grepTitle = test.titlePath().join(' '); - if (grepInvertMatcher?.(grepTitle)) - return false; - return grepMatcher(grepTitle) && testTitleMatcher(grepTitle); - }; - - const projectSuite = new Suite(project.name, 'project'); - projectSuite._projectConfig = project; - if (project._fullyParallel) - projectSuite._parallelMode = 'parallel'; - rootSuite._addSuite(projectSuite); - for (const file of files) { - const fileSuite = fileSuites.get(file); - if (!fileSuite) - continue; - for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) { - const builtSuite = buildFileSuiteForProject(project, fileSuite, repeatEachIndex); - if (!filterTestsRemoveEmptySuites(builtSuite, titleMatcher)) - continue; - projectSuite._addSuite(builtSuite); - } - } - } - return rootSuite; -} export function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) { for (const child of suite.suites) { @@ -141,3 +103,19 @@ export function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites: } return false; } + +export function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[]) { + if (!focusedTestFileLines.length) + return; + const matchers = focusedTestFileLines.map(createFileMatcherFromFilter); + const testFileLineMatches = (testFileName: string, testLine: number, testColumn: number) => matchers.some(m => m(testFileName, testLine, testColumn)); + const suiteFilter = (suite: Suite) => !!suite.location && testFileLineMatches(suite.location.file, suite.location.line, suite.location.column); + const testFilter = (test: TestCase) => testFileLineMatches(test.location.file, test.location.line, test.location.column); + return filterSuite(suite, suiteFilter, testFilter); +} + +function createFileMatcherFromFilter(filter: TestFileFilter) { + const fileMatcher = createFileMatcher(filter.re || filter.exact || ''); + return (testFileName: string, testLine: number, testColumn: number) => + fileMatcher(testFileName) && (filter.line === testLine || filter.line === null) && (filter.column === testColumn || filter.column === null); +} diff --git a/packages/playwright-test/src/common/test.ts b/packages/playwright-test/src/common/test.ts index 28f4ef5a5f..b141a4a558 100644 --- a/packages/playwright-test/src/common/test.ts +++ b/packages/playwright-test/src/common/test.ts @@ -75,6 +75,11 @@ export class Suite extends Base implements reporterTypes.Suite { this._entries.push(suite); } + _prependSuite(suite: Suite) { + suite.parent = this; + this._entries.unshift(suite); + } + allTests(): TestCase[] { const result: TestCase[] = []; const visit = (suite: Suite) => { diff --git a/packages/playwright-test/src/common/types.ts b/packages/playwright-test/src/common/types.ts index 14a6a6a736..1f4d5db087 100644 --- a/packages/playwright-test/src/common/types.ts +++ b/packages/playwright-test/src/common/types.ts @@ -72,6 +72,8 @@ export interface FullProjectInternal extends FullProjectPublic { _fullyParallel: boolean; _expect: Project['expect']; _respectGitIgnore: boolean; + _deps: string[]; + _depProjects: FullProjectInternal[]; snapshotPathTemplate: string; } diff --git a/packages/playwright-test/src/runner/loadUtils.ts b/packages/playwright-test/src/runner/loadUtils.ts index 25a62de9fd..a08df03729 100644 --- a/packages/playwright-test/src/runner/loadUtils.ts +++ b/packages/playwright-test/src/runner/loadUtils.ts @@ -19,56 +19,139 @@ import type { Reporter, TestError } from '../../types/testReporter'; import type { LoadError } from '../common/fixtures'; import { LoaderHost } from './loaderHost'; import type { Multiplexer } from '../reporters/multiplexer'; -import { createRootSuite, filterOnly, filterSuite } from '../common/suiteUtils'; -import type { Suite, TestCase } from '../common/test'; +import { Suite } from '../common/test'; +import type { TestCase } from '../common/test'; import { loadTestFilesInProcess } from '../common/testLoader'; import type { FullConfigInternal, FullProjectInternal } from '../common/types'; -import { errorWithFile } from '../util'; +import { createFileMatcherFromFilters, createTitleMatcher, errorWithFile } from '../util'; import type { Matcher, TestFileFilter } from '../util'; -import { createFileMatcher } from '../util'; -import { collectFilesForProject, filterProjects } from './projectUtils'; +import { collectFilesForProject, filterProjects, projectsThatAreDependencies } from './projectUtils'; import { requireOrImport } from '../common/transform'; import { serializeConfig } from '../common/ipc'; +import { buildFileSuiteForProject, filterByFocusedLine, filterOnly, filterTestsRemoveEmptySuites } from '../common/suiteUtils'; +import { filterForShard } from './testGroups'; type LoadOptions = { listOnly: boolean; testFileFilters: TestFileFilter[]; - testTitleMatcher: Matcher; + testTitleMatcher?: Matcher; projectFilter?: string[]; passWithNoTests?: boolean; }; export async function loadAllTests(config: FullConfigInternal, reporter: Multiplexer, options: LoadOptions, errors: TestError[]): Promise { const projects = filterProjects(config.projects, options.projectFilter); - const filesByProject = new Map(); - const allTestFiles = new Set(); - for (const project of projects) { - const files = await collectFilesForProject(project, options.testFileFilters); - filesByProject.set(project, files); - files.forEach(file => allTestFiles.add(file)); + + let filesToRunByProject = new Map(); + let topLevelProjects: FullProjectInternal[]; + let dependencyProjects: FullProjectInternal[]; + { + // First collect all files for the projects in the command line, don't apply any file filters. + const allFilesForProject = new Map(); + for (const project of projects) { + const files = await collectFilesForProject(project); + allFilesForProject.set(project, files); + } + + // Filter files based on the file filters, eliminate the empty projects. + const commandLineFileMatcher = options.testFileFilters.length ? createFileMatcherFromFilters(options.testFileFilters) : null; + for (const [project, files] of allFilesForProject) { + const filteredFiles = commandLineFileMatcher ? files.filter(commandLineFileMatcher) : files; + if (filteredFiles.length) + filesToRunByProject.set(project, filteredFiles); + } + // Remove dependency projects, they'll be added back later. + for (const project of projectsThatAreDependencies([...filesToRunByProject.keys()])) + filesToRunByProject.delete(project); + + // Shard only the top-level projects. + if (config.shard) + filesToRunByProject = filterForShard(config.shard, filesToRunByProject); + + // Re-build the closure, project set might have changed. + topLevelProjects = [...filesToRunByProject.keys()]; + dependencyProjects = projectsThatAreDependencies(topLevelProjects); + + // (Re-)add all files for dependent projects, disregard filters. + for (const project of dependencyProjects) { + const files = allFilesForProject.get(project) || await collectFilesForProject(project); + filesToRunByProject.set(project, files); + } } - // Load all tests. + // Load all test files and create a preprocessed root. Child suites are files there. + const allTestFiles = new Set(); + for (const files of filesToRunByProject.values()) + files.forEach(file => allTestFiles.add(file)); const preprocessRoot = await loadTests(config, reporter, allTestFiles, errors); // Complain about duplicate titles. errors.push(...createDuplicateTitlesErrors(config, preprocessRoot)); - // Filter tests to respect line/column filter. - filterByFocusedLine(preprocessRoot, options.testFileFilters); + // Create root suites with clones for the projects. + const rootSuite = new Suite('', 'root'); + + // First iterate leaf projects to focus only, then add all other projects. + for (const project of topLevelProjects) { + const projectSuite = await createProjectSuite(preprocessRoot, project, options, filesToRunByProject.get(project)!); + if (projectSuite) + rootSuite._addSuite(projectSuite); + } // Complain about only. if (config.forbidOnly) { - const onlyTestsAndSuites = preprocessRoot._getOnlyItems(); + const onlyTestsAndSuites = rootSuite._getOnlyItems(); if (onlyTestsAndSuites.length > 0) errors.push(...createForbidOnlyErrors(onlyTestsAndSuites)); } - // Filter only. - if (!options.listOnly) - filterOnly(preprocessRoot); + // Filter only for leaf projects. + filterOnly(rootSuite); - return await createRootSuite(preprocessRoot, options.testTitleMatcher, filesByProject); + // Prepend the projects that are dependencies. + for (const project of dependencyProjects) { + const projectSuite = await createProjectSuite(preprocessRoot, project, { ...options, testFileFilters: [], testTitleMatcher: undefined }, filesToRunByProject.get(project)!); + if (projectSuite) + rootSuite._prependSuite(projectSuite); + } + + return rootSuite; +} + +async function createProjectSuite(preprocessRoot: Suite, project: FullProjectInternal, options: LoadOptions, files: string[]): Promise { + const fileSuites = new Map(); + for (const fileSuite of preprocessRoot.suites) + fileSuites.set(fileSuite._requireFile, fileSuite); + + const projectSuite = new Suite(project.name, 'project'); + projectSuite._projectConfig = project; + if (project._fullyParallel) + projectSuite._parallelMode = 'parallel'; + for (const file of files) { + const fileSuite = fileSuites.get(file); + if (!fileSuite) + continue; + for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) { + const builtSuite = buildFileSuiteForProject(project, fileSuite, repeatEachIndex); + projectSuite._addSuite(builtSuite); + } + } + // Filter tests to respect line/column filter. + filterByFocusedLine(projectSuite, options.testFileFilters); + + const grepMatcher = createTitleMatcher(project.grep); + const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null; + + const titleMatcher = (test: TestCase) => { + const grepTitle = test.titlePath().join(' '); + if (grepInvertMatcher?.(grepTitle)) + return false; + return grepMatcher(grepTitle) && (!options.testTitleMatcher || options.testTitleMatcher(grepTitle)); + }; + + if (filterTestsRemoveEmptySuites(projectSuite, titleMatcher)) + return projectSuite; + return null; } async function loadTests(config: FullConfigInternal, reporter: Multiplexer, testFiles: Set, errors: TestError[]): Promise { @@ -89,22 +172,6 @@ async function loadTests(config: FullConfigInternal, reporter: Multiplexer, test } } -function createFileMatcherFromFilter(filter: TestFileFilter) { - const fileMatcher = createFileMatcher(filter.re || filter.exact || ''); - return (testFileName: string, testLine: number, testColumn: number) => - fileMatcher(testFileName) && (filter.line === testLine || filter.line === null) && (filter.column === testColumn || filter.column === null); -} - -function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[]) { - if (!focusedTestFileLines.length) - return; - const matchers = focusedTestFileLines.map(createFileMatcherFromFilter); - const testFileLineMatches = (testFileName: string, testLine: number, testColumn: number) => matchers.some(m => m(testFileName, testLine, testColumn)); - const suiteFilter = (suite: Suite) => !!suite.location && testFileLineMatches(suite.location.file, suite.location.line, suite.location.column); - const testFilter = (test: TestCase) => testFileLineMatches(test.location.file, test.location.line, test.location.column); - return filterSuite(suite, suiteFilter, testFilter); -} - function createForbidOnlyErrors(onlyTestsAndSuites: (TestCase | Suite)[]): TestError[] { const errors: TestError[] = []; for (const testOrSuite of onlyTestsAndSuites) { diff --git a/packages/playwright-test/src/runner/projectUtils.ts b/packages/playwright-test/src/runner/projectUtils.ts index 750b90227d..331c9a541d 100644 --- a/packages/playwright-test/src/runner/projectUtils.ts +++ b/packages/playwright-test/src/runner/projectUtils.ts @@ -19,8 +19,7 @@ import path from 'path'; import { minimatch } from 'playwright-core/lib/utilsBundle'; import { promisify } from 'util'; import type { FullProjectInternal } from '../common/types'; -import type { TestFileFilter } from '../util'; -import { createFileMatcher, createFileMatcherFromFilters } from '../util'; +import { createFileMatcher } from '../util'; const readFileAsync = promisify(fs.readFile); const readDirAsync = promisify(fs.readdir); @@ -50,17 +49,33 @@ export function filterProjects(projects: FullProjectInternal[], projectNames?: s return result; } -export async function collectFilesForProject(project: FullProjectInternal, commandLineFileFilters: TestFileFilter[]): Promise { +export function projectsThatAreDependencies(projects: FullProjectInternal[]): FullProjectInternal[] { + const result = new Set(); + const visit = (depth: number, project: FullProjectInternal) => { + if (depth > 100) { + const error = new Error('Circular dependency detected between projects.'); + error.stack = ''; + throw error; + } + if (result.has(project)) + return; + project._depProjects.map(visit.bind(undefined, depth + 1)); + project._depProjects.forEach(dep => result.add(dep)); + }; + projects.forEach(visit.bind(undefined, 0)); + return [...result]; +} + +export async function collectFilesForProject(project: FullProjectInternal): Promise { const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx']; const testFileExtension = (file: string) => extensions.includes(path.extname(file)); - const commandLineFileMatcher = commandLineFileFilters.length ? createFileMatcherFromFilters(commandLineFileFilters) : () => true; const allFiles = await collectFiles(project.testDir, project._respectGitIgnore); const testMatch = createFileMatcher(project.testMatch); const testIgnore = createFileMatcher(project.testIgnore); const testFiles = allFiles.filter(file => { if (!testFileExtension(file)) return false; - const isTest = !testIgnore(file) && testMatch(file) && commandLineFileMatcher(file); + const isTest = !testIgnore(file) && testMatch(file); if (!isTest) return false; return true; diff --git a/packages/playwright-test/src/runner/runner.ts b/packages/playwright-test/src/runner/runner.ts index 7570e9b503..e5cd3b2ed7 100644 --- a/packages/playwright-test/src/runner/runner.ts +++ b/packages/playwright-test/src/runner/runner.ts @@ -50,7 +50,7 @@ export class Runner { for (const project of projects) { report.projects.push({ ...sanitizeConfigForJSON(project, new Set()), - files: await collectFilesForProject(project, []) + files: await collectFilesForProject(project) }); } return report; @@ -74,7 +74,7 @@ export class Runner { options, reporter, plugins: [], - testGroups: [], + phases: [], }; reporter.onConfigure(config); @@ -90,7 +90,7 @@ export class Runner { const taskStatus = await taskRunner.run(context, deadline); let status: FullResult['status'] = 'passed'; - if (context.dispatcher?.hasWorkerErrors() || context.rootSuite?.allTests().some(test => !test.ok())) + if (context.phases.find(p => p.dispatcher.hasWorkerErrors()) || context.rootSuite?.allTests().some(test => !test.ok())) status = 'failed'; if (status === 'passed' && taskStatus !== 'passed') status = taskStatus; diff --git a/packages/playwright-test/src/runner/tasks.ts b/packages/playwright-test/src/runner/tasks.ts index f031c0e99d..deeaa5256f 100644 --- a/packages/playwright-test/src/runner/tasks.ts +++ b/packages/playwright-test/src/runner/tasks.ts @@ -17,16 +17,16 @@ import fs from 'fs'; import path from 'path'; import { promisify } from 'util'; -import { rimraf } from 'playwright-core/lib/utilsBundle'; +import { debug, rimraf } from 'playwright-core/lib/utilsBundle'; import { Dispatcher } from './dispatcher'; import type { TestRunnerPlugin, TestRunnerPluginRegistration } from '../plugins'; import type { Multiplexer } from '../reporters/multiplexer'; import type { TestGroup } from '../runner/testGroups'; -import { createTestGroups, filterForShard } from '../runner/testGroups'; +import { createTestGroups } from '../runner/testGroups'; import type { Task } from './taskRunner'; import { TaskRunner } from './taskRunner'; import type { Suite } from '../common/test'; -import type { FullConfigInternal } from '../common/types'; +import type { FullConfigInternal, FullProjectInternal } from '../common/types'; import { loadAllTests, loadGlobalHook } from './loadUtils'; import type { Matcher, TestFileFilter } from '../util'; @@ -41,14 +41,22 @@ type TaskRunnerOptions = { passWithNoTests?: boolean; }; +type ProjectWithTestGroups = { + project: FullProjectInternal; + projectSuite: Suite; + testGroups: TestGroup[]; +}; + export type TaskRunnerState = { options: TaskRunnerOptions; reporter: Multiplexer; config: FullConfigInternal; plugins: TestRunnerPlugin[]; - testGroups: TestGroup[]; rootSuite?: Suite; - dispatcher?: Dispatcher; + phases: { + dispatcher: Dispatcher, + projects: ProjectWithTestGroups[] + }[]; }; export function createTaskRunner(config: FullConfigInternal, reporter: Multiplexer): TaskRunner { @@ -71,8 +79,7 @@ export function createTaskRunner(config: FullConfigInternal, reporter: Multiplex return () => reporter.onEnd(); }); - taskRunner.addTask('setup workers', createSetupWorkersTask()); - taskRunner.addTask('test suite', async ({ dispatcher, testGroups }) => dispatcher!.run(testGroups)); + taskRunner.addTask('test suite', createRunTestsTask()); return taskRunner; } @@ -113,17 +120,6 @@ function createGlobalSetupTask(): Task { }; } -function createSetupWorkersTask(): Task { - return async params => { - const { config, reporter } = params; - const dispatcher = new Dispatcher(config, reporter); - params.dispatcher = dispatcher; - return async () => { - await dispatcher.stop(); - }; - }; -} - function createRemoveOutputDirsTask(): Task { return async ({ config, options }) => { const outputDirs = new Set(); @@ -151,20 +147,95 @@ function createLoadTask(): Task { const { config, reporter, options } = context; context.rootSuite = await loadAllTests(config, reporter, options, errors); // Fail when no tests. - if (!context.rootSuite.allTests().length && !context.options.passWithNoTests) + if (!context.rootSuite.allTests().length && !context.options.passWithNoTests && !config.shard) throw new Error(`No tests found`); }; } function createTestGroupsTask(): Task { return async context => { - const { config, rootSuite } = context; + const { config, rootSuite, reporter } = context; + for (const phase of buildPhases(rootSuite!.suites)) { + // Go over the phases, for each phase create list of task groups. + const projects: ProjectWithTestGroups[] = []; + for (const projectSuite of phase) { + const testGroups = createTestGroups(projectSuite, config.workers); + projects.push({ + project: projectSuite._projectConfig!, + projectSuite, + testGroups, + }); + } - for (const projectSuite of rootSuite!.suites) - context.testGroups.push(...createTestGroups(projectSuite, config.workers)); + const testGroupsInPhase = projects.reduce((acc, project) => acc + project.testGroups.length, 0); + debug('pw:test:task')(`running phase with ${projects.map(p => p.project.name).sort()} projects, ${testGroupsInPhase} testGroups`); + context.phases.push({ dispatcher: new Dispatcher(config, reporter), projects }); + context.config._maxConcurrentTestGroups = Math.max(context.config._maxConcurrentTestGroups, testGroupsInPhase); + } - if (context.config.shard) - filterForShard(context.config.shard, rootSuite!, context.testGroups); - context.config._maxConcurrentTestGroups = context.testGroups.length; + return async () => { + for (const { dispatcher } of context.phases.reverse()) + await dispatcher.stop(); + }; }; } + +function createRunTestsTask(): Task { + return async context => { + const { phases } = context; + const successfulProjects = new Set(); + + for (const { dispatcher, projects } of phases) { + // Each phase contains dispatcher and a set of test groups. + // We don't want to run the test groups beloning to the projects + // that depend on the projects that failed previously. + const phaseTestGroups: TestGroup[] = []; + for (const { project, testGroups } of projects) { + const hasFailedDeps = project._depProjects.some(p => !successfulProjects.has(p)); + if (!hasFailedDeps) { + phaseTestGroups.push(...testGroups); + } else { + for (const testGroup of testGroups) { + for (const test of testGroup.tests) + test._appendTestResult().status = 'skipped'; + } + } + } + + if (phaseTestGroups.length) { + await dispatcher!.run(phaseTestGroups); + await dispatcher.stop(); + } + + // If the worker broke, fail everything, we have no way of knowing which + // projects failed. + if (!dispatcher.hasWorkerErrors()) { + for (const { project, projectSuite } of projects) { + const hasFailedDeps = project._depProjects.some(p => !successfulProjects.has(p)); + if (!hasFailedDeps && !projectSuite.allTests().some(test => !test.ok())) + successfulProjects.add(project); + } + } + } + }; +} + +function buildPhases(projectSuites: Suite[]): Suite[][] { + const phases: Suite[][] = []; + const processed = new Set(); + for (let i = 0; i < projectSuites.length; i++) { + const phase: Suite[] = []; + for (const projectSuite of projectSuites) { + if (processed.has(projectSuite._projectConfig!)) + continue; + if (projectSuite._projectConfig!._depProjects.find(p => !processed.has(p))) + continue; + phase.push(projectSuite); + } + for (const projectSuite of phase) + processed.add(projectSuite._projectConfig!); + if (phase.length) + phases.push(phase); + } + return phases; +} diff --git a/packages/playwright-test/src/runner/testGroups.ts b/packages/playwright-test/src/runner/testGroups.ts index 0497fd4b74..ceb60a7310 100644 --- a/packages/playwright-test/src/runner/testGroups.ts +++ b/packages/playwright-test/src/runner/testGroups.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { filterSuiteWithOnlySemantics } from '../common/suiteUtils'; import type { Suite, TestCase } from '../common/test'; +import type { FullProjectInternal } from '../common/types'; export type TestGroup = { workerHash: string; @@ -131,15 +131,10 @@ export function createTestGroups(projectSuite: Suite, workers: number): TestGrou return result; } -export async function filterForShard(shard: { total: number, current: number }, rootSuite: Suite, testGroups: TestGroup[]) { - // Each shard includes: - // - its portion of the regular tests - // - project setup tests for the projects that have regular tests in this shard +export function filterForShard(shard: { total: number, current: number }, filesByProject: Map): Map { let shardableTotal = 0; - for (const group of testGroups) - shardableTotal += group.tests.length; - - const shardTests = new Set(); + for (const files of filesByProject.values()) + shardableTotal += files.length; // Each shard gets some tests. const shardSize = Math.floor(shardableTotal / shard.total); @@ -150,27 +145,16 @@ export async function filterForShard(shard: { total: number, current: number }, const from = shardSize * currentShard + Math.min(extraOne, currentShard); const to = from + shardSize + (currentShard < extraOne ? 1 : 0); let current = 0; - const shardProjects = new Set(); - const shardTestGroups = []; - for (const group of testGroups) { - // Any test group goes to the shard that contains the first test of this group. - // So, this shard gets any group that starts at [from; to) - if (current >= from && current < to) { - shardProjects.add(group.projectId); - shardTestGroups.push(group); - for (const test of group.tests) - shardTests.add(test); + const result = new Map(); + for (const [project, files] of filesByProject) { + const shardFiles: string[] = []; + for (const file of files) { + if (current >= from && current < to) + shardFiles.push(file); + ++current; } - current += group.tests.length; - } - testGroups.length = 0; - testGroups.push(...shardTestGroups); - - if (!shardTests.size) { - // Filtering with "only semantics" does not work when we have zero tests - it leaves all the tests. - // We need an empty suite in this case. - rootSuite._entries = []; - } else { - filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test)); + if (shardFiles.length) + result.set(project, shardFiles); } + return result; } diff --git a/tests/playwright-test/deps.spec.ts b/tests/playwright-test/deps.spec.ts new file mode 100644 index 0000000000..cce7569d97 --- /dev/null +++ b/tests/playwright-test/deps.spec.ts @@ -0,0 +1,245 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect, stripAnsi } from './playwright-test-fixtures'; + +test('should run projects with dependencies', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'A' }, + { name: 'B', _deps: ['A'] }, + { name: 'C', _deps: ['A'] }, + ], + };`, + 'test.spec.ts': ` + const { test } = pwt; + test('test', async ({}, testInfo) => { + console.log('\\n%%' + testInfo.project.name); + }); + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(3); + expect(extractLines(result.output)).toEqual(['A', 'B', 'C']); +}); + +test('should not run project if dependency failed', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'A' }, + { name: 'B', _deps: ['A'] }, + { name: 'C', _deps: ['B'] }, + ], + };`, + 'test.spec.ts': ` + const { test } = pwt; + test('test', async ({}, testInfo) => { + console.log('\\n%%' + testInfo.project.name); + if (testInfo.project.name === 'B') + throw new Error('Failed project B'); + }); + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(1); + expect(result.failed).toBe(1); + expect(result.skipped).toBe(1); + expect(result.output).toContain('Failed project B'); + expect(extractLines(result.output)).toEqual(['A', 'B']); +}); + +test('should not run project if dependency failed (2)', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'A1' }, + { name: 'A2', _deps: ['A1'] }, + { name: 'A3', _deps: ['A2'] }, + { name: 'B1' }, + { name: 'B2', _deps: ['B1'] }, + { name: 'B3', _deps: ['B2'] }, + ], + };`, + 'test.spec.ts': ` + const { test } = pwt; + test('test', async ({}, testInfo) => { + console.log('\\n%%' + testInfo.project.name); + if (testInfo.project.name === 'B1') + throw new Error('Failed project B1'); + }); + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(1); + expect(extractLines(result.output).sort()).toEqual(['A1', 'A2', 'A3', 'B1']); +}); + +test('should filter by project list, but run deps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'A' }, + { name: 'B' }, + { name: 'C', _deps: ['A'] }, + { name: 'D' }, + ] }; + `, + 'test.spec.ts': ` + const { test } = pwt; + test('pass', async ({}, testInfo) => { + console.log('\\n%%' + testInfo.project.name); + }); + ` + }, { project: ['C', 'D'] }); + expect(result.passed).toBe(3); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(0); + expect(extractLines(result.output).sort()).toEqual(['A', 'C', 'D']); +}); + + +test('should not filter dependency by file name', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'A' }, + { name: 'B', _deps: ['A'] }, + ] }; + `, + 'one.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + 'two.spec.ts': `pwt.test('pass', () => { });`, + }, undefined, undefined, { additionalArgs: ['two.spec.ts'] }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toContain('1) [A] › one.spec.ts:4:5 › fails'); +}); + +test('should not filter dependency by only', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'setup', testMatch: /setup.ts/ }, + { name: 'browser', _deps: ['setup'] }, + ] }; + `, + 'setup.ts': ` + pwt.test('passes', () => { + console.log('\\n%% setup in ' + pwt.test.info().project.name); + }); + pwt.test.only('passes 2', () => { + console.log('\\n%% setup 2 in ' + pwt.test.info().project.name); + }); + `, + 'test.spec.ts': `pwt.test('pass', () => { + console.log('\\n%% test in ' + pwt.test.info().project.name); + });`, + }); + expect(result.exitCode).toBe(0); + expect(extractLines(result.output)).toEqual(['setup in setup', 'setup 2 in setup', 'test in browser']); +}); + +test('should not filter dependency by only 2', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'setup', testMatch: /setup.ts/ }, + { name: 'browser', _deps: ['setup'] }, + ] }; + `, + 'setup.ts': ` + pwt.test('passes', () => { + console.log('\\n%% setup in ' + pwt.test.info().project.name); + }); + pwt.test.only('passes 2', () => { + console.log('\\n%% setup 2 in ' + pwt.test.info().project.name); + }); + `, + 'test.spec.ts': `pwt.test('pass', () => { + console.log('\\n%% test in ' + pwt.test.info().project.name); + });`, + }, { project: ['setup'] }); + expect(result.exitCode).toBe(0); + expect(extractLines(result.output)).toEqual(['setup 2 in setup']); +}); + +test('should not filter dependency by only 3', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'setup', testMatch: /setup.*.ts/ }, + { name: 'browser', _deps: ['setup'] }, + ] }; + `, + 'setup-1.ts': ` + pwt.test('setup 1', () => { + console.log('\\n%% setup in ' + pwt.test.info().project.name); + }); + `, + 'setup-2.ts': ` + pwt.test('setup 2', () => { + console.log('\\n%% setup 2 in ' + pwt.test.info().project.name); + }); + `, + 'test.spec.ts': `pwt.test('pass', () => { + console.log('\\n%% test in ' + pwt.test.info().project.name); + });`, + }, undefined, undefined, { additionalArgs: ['setup-2.ts'] }); + expect(result.exitCode).toBe(0); + expect(extractLines(result.output)).toEqual(['setup 2 in setup']); +}); + +test('should report skipped dependent tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'setup', testMatch: /setup.ts/ }, + { name: 'browser', _deps: ['setup'] }, + ] }; + `, + 'setup.ts': ` + pwt.test('setup', () => { + expect(1).toBe(2); + }); + `, + 'test.spec.ts': `pwt.test('pass', () => {});`, + }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(0); + expect(result.skipped).toBe(1); + expect(result.results.length).toBe(2); +}); + +test('should report circular dependencies', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'A', _deps: ['B'] }, + { name: 'B', _deps: ['A'] }, + ] }; + `, + 'test.spec.ts': `pwt.test('pass', () => {});`, + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Circular dependency detected between projects.'); +}); + +function extractLines(output: string): string[] { + return stripAnsi(output).split('\n').filter(line => line.startsWith('%%')).map(line => line.substring(2).trim()); +} diff --git a/tests/playwright-test/global-scripts.spec.ts b/tests/playwright-test/global-scripts.spec.ts index c44c9ee308..8775e4a208 100644 --- a/tests/playwright-test/global-scripts.spec.ts +++ b/tests/playwright-test/global-scripts.spec.ts @@ -16,8 +16,6 @@ import path from 'path'; import { test, expect } from './playwright-test-fixtures'; -test.fixme(true, 'Restore this'); - type Timeline = { titlePath: string[], event: 'begin' | 'end' }[]; function formatTimeline(timeline: Timeline) { @@ -70,11 +68,15 @@ test('should work for one project', async ({ runGroups }, testInfo) => { const files = { 'playwright.config.ts': ` module.exports = { - globalScripts: /.*global.ts/, projects: [ + { + name: 'setup', + testMatch: /.*global.ts/, + }, { name: 'p1', testMatch: /.*.test.ts/, + _deps: ['setup'], }, ] };`, @@ -92,10 +94,10 @@ test('should work for one project', async ({ runGroups }, testInfo) => { const { exitCode, passed, timeline } = await runGroups(files); expect(exitCode).toBe(0); expect(passed).toBe(4); - expect(formatTimeline(timeline)).toEqual(`Global Scripts > global.ts > setup1 [begin] -Global Scripts > global.ts > setup1 [end] -Global Scripts > global.ts > setup2 [begin] -Global Scripts > global.ts > setup2 [end] + expect(formatTimeline(timeline)).toEqual(`setup > global.ts > setup1 [begin] +setup > global.ts > setup1 [end] +setup > global.ts > setup2 [begin] +setup > global.ts > setup2 [end] p1 > a.test.ts > test1 [begin] p1 > a.test.ts > test1 [end] p1 > a.test.ts > test2 [begin] @@ -106,15 +108,20 @@ test('should work for several projects', async ({ runGroups }, testInfo) => { const files = { 'playwright.config.ts': ` module.exports = { - globalScripts: /.*global.ts/, projects: [ + { + name: 'setup', + testMatch: /.*global.ts/, + }, { name: 'p1', testMatch: /.*a.test.ts/, + _deps: ['setup'], }, { name: 'p2', testMatch: /.*b.test.ts/, + _deps: ['setup'], }, ] };`, @@ -144,15 +151,20 @@ test('should skip tests if global setup fails', async ({ runGroups }, testInfo) const files = { 'playwright.config.ts': ` module.exports = { - globalScripts: /.*global.ts/, projects: [ + { + name: 'setup', + testMatch: /.*global.ts/, + }, { name: 'p1', testMatch: /.*a.test.ts/, + _deps: ['setup'], }, { name: 'p2', testMatch: /.*b.test.ts/, + _deps: ['setup'], }, ] };`, @@ -181,10 +193,14 @@ test('should run setup in each project shard', async ({ runGroups }, testInfo) = const files = { 'playwright.config.ts': ` module.exports = { - globalScripts: /.*global.ts/, projects: [ + { + name: 'setup', + testMatch: /.*global.ts/, + }, { name: 'p1', + _deps: ['setup'], }, ] };`, diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 558dac7b36..980ad3516b 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -24,10 +24,7 @@ test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => testDir: '..', globalSetup: './globalSetup', globalTeardown: path.join(__dirname, 'globalTeardown.ts'), - projects: [ - { name: 'p1' }, - { name: 'p2' }, - ] + projects: [{ name: 'p1' }] }; `, 'dir/globalSetup.ts': ` @@ -46,7 +43,7 @@ test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => console.log('\\n%%from-test'); }); `, - }, { 'project': 'p2', 'config': 'dir' }); + }, { 'config': 'dir' }); expect(result.passed).toBe(1); expect(result.failed).toBe(0); expect(stripAnsi(result.output).split('\n').filter(line => line.startsWith('%%'))).toEqual([ diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.ts index 8eb018e8bd..625021c123 100644 --- a/tests/playwright-test/shard.spec.ts +++ b/tests/playwright-test/shard.spec.ts @@ -21,23 +21,32 @@ const tests = { export const headlessTest = pwt.test.extend({ headless: false }); export const headedTest = pwt.test.extend({ headless: true }); `, - 'a.spec.ts': ` + 'a1.spec.ts': ` import { headlessTest, headedTest } from './helper'; headlessTest('test1', async () => { console.log('test1-done'); }); + `, + 'a2.spec.ts': ` + import { headlessTest, headedTest } from './helper'; headedTest('test2', async () => { console.log('test2-done'); }); + `, + 'a3.spec.ts': ` + import { headlessTest, headedTest } from './helper'; headlessTest('test3', async () => { console.log('test3-done'); }); `, - 'b.spec.ts': ` + 'b1.spec.ts': ` import { headlessTest, headedTest } from './helper'; headlessTest('test4', async () => { console.log('test4-done'); }); + `, + 'b2.spec.ts': ` + import { headlessTest, headedTest } from './helper'; headedTest('test5', async () => { console.log('test5-done'); }); @@ -50,8 +59,8 @@ test('should respect shard=1/2', async ({ runInlineTest }) => { expect(result.passed).toBe(3); expect(result.skipped).toBe(0); expect(result.output).toContain('test1-done'); + expect(result.output).toContain('test2-done'); expect(result.output).toContain('test3-done'); - expect(result.output).toContain('test4-done'); }); test('should respect shard=2/2', async ({ runInlineTest }) => { @@ -59,7 +68,7 @@ test('should respect shard=2/2', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); expect(result.passed).toBe(2); expect(result.skipped).toBe(0); - expect(result.output).toContain('test2-done'); + expect(result.output).toContain('test4-done'); expect(result.output).toContain('test5-done'); }); @@ -83,6 +92,6 @@ test('should respect shard=1/2 in config', async ({ runInlineTest }) => { expect(result.passed).toBe(3); expect(result.skipped).toBe(0); expect(result.output).toContain('test1-done'); + expect(result.output).toContain('test2-done'); expect(result.output).toContain('test3-done'); - expect(result.output).toContain('test4-done'); });