Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore: project deps (#20514)

Browse source
This commit is contained in:
Pavel Feldman 2023-01-31 15:59:13 -08:00 committed by GitHub
parent 9c6c31a442
commit 08e4b50ff6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 566 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
packages/playwright-test/src/common/configLoader.ts
View file

@ -146,6 +146,8 @@ export class ConfigLoader {
}
this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata);
this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath));
resolveProjectDependencies(this._fullConfig.projects);
this._assignUniqueProjectIds(this._fullConfig.projects);
}
@ -216,6 +218,8 @@ export class ConfigLoader {
testMatch: takeFirst(projectConfig.testMatch, config.testMatch, '**/?(*.)@(spec|test).*'),
timeout: takeFirst(projectConfig.timeout, config.timeout, defaultTimeout),
use: mergeObjects(config.use, projectConfig.use),
_deps: (projectConfig as any)._deps || [],
_depProjects: [],
};
}
}
@ -454,6 +458,19 @@ function resolveScript(id: string, rootDir: string) {
return require.resolve(id, { paths: [rootDir] });
}
function resolveProjectDependencies(projects: FullProjectInternal[]) {
for (const project of projects) {
for (const dependencyName of project._deps) {
const dependencies = projects.filter(p => p.name === dependencyName);
if (!dependencies.length)
throw new Error(`Project '${project.name}' depends on unknown project '${dependencyName}'`);
if (dependencies.length > 1)
throw new Error(`Project dependencies should have unique names, reading ${dependencyName}`);
project._depProjects.push(...dependencies);
}
}
}
export const kDefaultConfigFiles = ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs'];
export function resolveConfigFile(configFileOrDirectory: string): string | null {

60
packages/playwright-test/src/common/suiteUtils.ts
View file

@ -16,49 +16,11 @@
import path from 'path';
import { calculateSha1 } from 'playwright-core/lib/utils';
import type { TestCase } from './test';
import { Suite } from './test';
import type { Suite, TestCase } from './test';
import type { FullProjectInternal } from './types';
import type { Matcher } from '../util';
import { createTitleMatcher } from '../util';
import type { TestFileFilter } from '../util';
import { createFileMatcher } from '../util';
export async function createRootSuite(preprocessRoot: Suite, testTitleMatcher: Matcher, filesByProject: Map<FullProjectInternal, string[]>): Promise<Suite> {
// Generate projects.
const fileSuites = new Map<string, Suite>();
for (const fileSuite of preprocessRoot.suites)
fileSuites.set(fileSuite._requireFile, fileSuite);
const rootSuite = new Suite('', 'root');
for (const [project, files] of filesByProject) {
const grepMatcher = createTitleMatcher(project.grep);
const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
const titleMatcher = (test: TestCase) => {
const grepTitle = test.titlePath().join(' ');
if (grepInvertMatcher?.(grepTitle))
return false;
return grepMatcher(grepTitle) && testTitleMatcher(grepTitle);
};
const projectSuite = new Suite(project.name, 'project');
projectSuite._projectConfig = project;
if (project._fullyParallel)
projectSuite._parallelMode = 'parallel';
rootSuite._addSuite(projectSuite);
for (const file of files) {
const fileSuite = fileSuites.get(file);
if (!fileSuite)
continue;
for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) {
const builtSuite = buildFileSuiteForProject(project, fileSuite, repeatEachIndex);
if (!filterTestsRemoveEmptySuites(builtSuite, titleMatcher))
continue;
projectSuite._addSuite(builtSuite);
}
}
}
return rootSuite;
}
export function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {
for (const child of suite.suites) {
@ -141,3 +103,19 @@ export function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites:
}
return false;
}
export function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[]) {
if (!focusedTestFileLines.length)
return;
const matchers = focusedTestFileLines.map(createFileMatcherFromFilter);
const testFileLineMatches = (testFileName: string, testLine: number, testColumn: number) => matchers.some(m => m(testFileName, testLine, testColumn));
const suiteFilter = (suite: Suite) => !!suite.location && testFileLineMatches(suite.location.file, suite.location.line, suite.location.column);
const testFilter = (test: TestCase) => testFileLineMatches(test.location.file, test.location.line, test.location.column);
return filterSuite(suite, suiteFilter, testFilter);
}
function createFileMatcherFromFilter(filter: TestFileFilter) {
const fileMatcher = createFileMatcher(filter.re || filter.exact || '');
return (testFileName: string, testLine: number, testColumn: number) =>
fileMatcher(testFileName) && (filter.line === testLine || filter.line === null) && (filter.column === testColumn || filter.column === null);
}

5
packages/playwright-test/src/common/test.ts
View file

@ -75,6 +75,11 @@ export class Suite extends Base implements reporterTypes.Suite {
this._entries.push(suite);
}
_prependSuite(suite: Suite) {
suite.parent = this;
this._entries.unshift(suite);
}
allTests(): TestCase[] {
const result: TestCase[] = [];
const visit = (suite: Suite) => {

2
packages/playwright-test/src/common/types.ts
View file

@ -72,6 +72,8 @@ export interface FullProjectInternal extends FullProjectPublic {
_fullyParallel: boolean;
_expect: Project['expect'];
_respectGitIgnore: boolean;
_deps: string[];
_depProjects: FullProjectInternal[];
snapshotPathTemplate: string;
}

139
packages/playwright-test/src/runner/loadUtils.ts
View file

@ -19,56 +19,139 @@ import type { Reporter, TestError } from '../../types/testReporter';
import type { LoadError } from '../common/fixtures';
import { LoaderHost } from './loaderHost';
import type { Multiplexer } from '../reporters/multiplexer';
import { createRootSuite, filterOnly, filterSuite } from '../common/suiteUtils';
import type { Suite, TestCase } from '../common/test';
import { Suite } from '../common/test';
import type { TestCase } from '../common/test';
import { loadTestFilesInProcess } from '../common/testLoader';
import type { FullConfigInternal, FullProjectInternal } from '../common/types';
import { errorWithFile } from '../util';
import { createFileMatcherFromFilters, createTitleMatcher, errorWithFile } from '../util';
import type { Matcher, TestFileFilter } from '../util';
import { createFileMatcher } from '../util';
import { collectFilesForProject, filterProjects } from './projectUtils';
import { collectFilesForProject, filterProjects, projectsThatAreDependencies } from './projectUtils';
import { requireOrImport } from '../common/transform';
import { serializeConfig } from '../common/ipc';
import { buildFileSuiteForProject, filterByFocusedLine, filterOnly, filterTestsRemoveEmptySuites } from '../common/suiteUtils';
import { filterForShard } from './testGroups';
type LoadOptions = {
listOnly: boolean;
testFileFilters: TestFileFilter[];
testTitleMatcher: Matcher;
testTitleMatcher?: Matcher;
projectFilter?: string[];
passWithNoTests?: boolean;
};
export async function loadAllTests(config: FullConfigInternal, reporter: Multiplexer, options: LoadOptions, errors: TestError[]): Promise<Suite> {
const projects = filterProjects(config.projects, options.projectFilter);
const filesByProject = new Map<FullProjectInternal, string[]>();
const allTestFiles = new Set<string>();
for (const project of projects) {
const files = await collectFilesForProject(project, options.testFileFilters);
filesByProject.set(project, files);
files.forEach(file => allTestFiles.add(file));
let filesToRunByProject = new Map<FullProjectInternal, string[]>();
let topLevelProjects: FullProjectInternal[];
let dependencyProjects: FullProjectInternal[];
{
// First collect all files for the projects in the command line, don't apply any file filters.
const allFilesForProject = new Map<FullProjectInternal, string[]>();
for (const project of projects) {
const files = await collectFilesForProject(project);
allFilesForProject.set(project, files);
}
// Filter files based on the file filters, eliminate the empty projects.
const commandLineFileMatcher = options.testFileFilters.length ? createFileMatcherFromFilters(options.testFileFilters) : null;
for (const [project, files] of allFilesForProject) {
const filteredFiles = commandLineFileMatcher ? files.filter(commandLineFileMatcher) : files;
if (filteredFiles.length)
filesToRunByProject.set(project, filteredFiles);
}
// Remove dependency projects, they'll be added back later.
for (const project of projectsThatAreDependencies([...filesToRunByProject.keys()]))
filesToRunByProject.delete(project);
// Shard only the top-level projects.
if (config.shard)
filesToRunByProject = filterForShard(config.shard, filesToRunByProject);
// Re-build the closure, project set might have changed.
topLevelProjects = [...filesToRunByProject.keys()];
dependencyProjects = projectsThatAreDependencies(topLevelProjects);
// (Re-)add all files for dependent projects, disregard filters.
for (const project of dependencyProjects) {
const files = allFilesForProject.get(project) || await collectFilesForProject(project);
filesToRunByProject.set(project, files);
}
}
// Load all tests.
// Load all test files and create a preprocessed root. Child suites are files there.
const allTestFiles = new Set<string>();
for (const files of filesToRunByProject.values())
files.forEach(file => allTestFiles.add(file));
const preprocessRoot = await loadTests(config, reporter, allTestFiles, errors);
// Complain about duplicate titles.
errors.push(...createDuplicateTitlesErrors(config, preprocessRoot));
// Filter tests to respect line/column filter.
filterByFocusedLine(preprocessRoot, options.testFileFilters);
// Create root suites with clones for the projects.
const rootSuite = new Suite('', 'root');
// First iterate leaf projects to focus only, then add all other projects.
for (const project of topLevelProjects) {
const projectSuite = await createProjectSuite(preprocessRoot, project, options, filesToRunByProject.get(project)!);
if (projectSuite)
rootSuite._addSuite(projectSuite);
}
// Complain about only.
if (config.forbidOnly) {
const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
const onlyTestsAndSuites = rootSuite._getOnlyItems();
if (onlyTestsAndSuites.length > 0)
errors.push(...createForbidOnlyErrors(onlyTestsAndSuites));
}
// Filter only.
if (!options.listOnly)
filterOnly(preprocessRoot);
// Filter only for leaf projects.
filterOnly(rootSuite);
return await createRootSuite(preprocessRoot, options.testTitleMatcher, filesByProject);
// Prepend the projects that are dependencies.
for (const project of dependencyProjects) {
const projectSuite = await createProjectSuite(preprocessRoot, project, { ...options, testFileFilters: [], testTitleMatcher: undefined }, filesToRunByProject.get(project)!);
if (projectSuite)
rootSuite._prependSuite(projectSuite);
}
return rootSuite;
}
async function createProjectSuite(preprocessRoot: Suite, project: FullProjectInternal, options: LoadOptions, files: string[]): Promise<Suite | null> {
const fileSuites = new Map<string, Suite>();
for (const fileSuite of preprocessRoot.suites)
fileSuites.set(fileSuite._requireFile, fileSuite);
const projectSuite = new Suite(project.name, 'project');
projectSuite._projectConfig = project;
if (project._fullyParallel)
projectSuite._parallelMode = 'parallel';
for (const file of files) {
const fileSuite = fileSuites.get(file);
if (!fileSuite)
continue;
for (let repeatEachIndex = 0; repeatEachIndex < project.repeatEach; repeatEachIndex++) {
const builtSuite = buildFileSuiteForProject(project, fileSuite, repeatEachIndex);
projectSuite._addSuite(builtSuite);
}
}
// Filter tests to respect line/column filter.
filterByFocusedLine(projectSuite, options.testFileFilters);
const grepMatcher = createTitleMatcher(project.grep);
const grepInvertMatcher = project.grepInvert ? createTitleMatcher(project.grepInvert) : null;
const titleMatcher = (test: TestCase) => {
const grepTitle = test.titlePath().join(' ');
if (grepInvertMatcher?.(grepTitle))
return false;
return grepMatcher(grepTitle) && (!options.testTitleMatcher || options.testTitleMatcher(grepTitle));
};
if (filterTestsRemoveEmptySuites(projectSuite, titleMatcher))
return projectSuite;
return null;
}
async function loadTests(config: FullConfigInternal, reporter: Multiplexer, testFiles: Set<string>, errors: TestError[]): Promise<Suite> {
@ -89,22 +172,6 @@ async function loadTests(config: FullConfigInternal, reporter: Multiplexer, test
}
}
function createFileMatcherFromFilter(filter: TestFileFilter) {
const fileMatcher = createFileMatcher(filter.re || filter.exact || '');
return (testFileName: string, testLine: number, testColumn: number) =>
fileMatcher(testFileName) && (filter.line === testLine || filter.line === null) && (filter.column === testColumn || filter.column === null);
}
function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[]) {
if (!focusedTestFileLines.length)
return;
const matchers = focusedTestFileLines.map(createFileMatcherFromFilter);
const testFileLineMatches = (testFileName: string, testLine: number, testColumn: number) => matchers.some(m => m(testFileName, testLine, testColumn));
const suiteFilter = (suite: Suite) => !!suite.location && testFileLineMatches(suite.location.file, suite.location.line, suite.location.column);
const testFilter = (test: TestCase) => testFileLineMatches(test.location.file, test.location.line, test.location.column);
return filterSuite(suite, suiteFilter, testFilter);
}
function createForbidOnlyErrors(onlyTestsAndSuites: (TestCase | Suite)[]): TestError[] {
const errors: TestError[] = [];
for (const testOrSuite of onlyTestsAndSuites) {

25
packages/playwright-test/src/runner/projectUtils.ts
View file

@ -19,8 +19,7 @@ import path from 'path';
import { minimatch } from 'playwright-core/lib/utilsBundle';
import { promisify } from 'util';
import type { FullProjectInternal } from '../common/types';
import type { TestFileFilter } from '../util';
import { createFileMatcher, createFileMatcherFromFilters } from '../util';
import { createFileMatcher } from '../util';
const readFileAsync = promisify(fs.readFile);
const readDirAsync = promisify(fs.readdir);
@ -50,17 +49,33 @@ export function filterProjects(projects: FullProjectInternal[], projectNames?: s
return result;
}
export async function collectFilesForProject(project: FullProjectInternal, commandLineFileFilters: TestFileFilter[]): Promise<string[]> {
export function projectsThatAreDependencies(projects: FullProjectInternal[]): FullProjectInternal[] {
const result = new Set<FullProjectInternal>();
const visit = (depth: number, project: FullProjectInternal) => {
if (depth > 100) {
const error = new Error('Circular dependency detected between projects.');
error.stack = '';
throw error;
}
if (result.has(project))
return;
project._depProjects.map(visit.bind(undefined, depth + 1));
project._depProjects.forEach(dep => result.add(dep));
};
projects.forEach(visit.bind(undefined, 0));
return [...result];
}
export async function collectFilesForProject(project: FullProjectInternal): Promise<string[]> {
const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
const testFileExtension = (file: string) => extensions.includes(path.extname(file));
const commandLineFileMatcher = commandLineFileFilters.length ? createFileMatcherFromFilters(commandLineFileFilters) : () => true;
const allFiles = await collectFiles(project.testDir, project._respectGitIgnore);
const testMatch = createFileMatcher(project.testMatch);
const testIgnore = createFileMatcher(project.testIgnore);
const testFiles = allFiles.filter(file => {
if (!testFileExtension(file))
return false;
const isTest = !testIgnore(file) && testMatch(file) && commandLineFileMatcher(file);
const isTest = !testIgnore(file) && testMatch(file);
if (!isTest)
return false;
return true;

6
packages/playwright-test/src/runner/runner.ts
View file

@ -50,7 +50,7 @@ export class Runner {
for (const project of projects) {
report.projects.push({
...sanitizeConfigForJSON(project, new Set()),
files: await collectFilesForProject(project, [])
files: await collectFilesForProject(project)
});
}
return report;
@ -74,7 +74,7 @@ export class Runner {
options,
reporter,
plugins: [],
testGroups: [],
phases: [],
};
reporter.onConfigure(config);
@ -90,7 +90,7 @@ export class Runner {
const taskStatus = await taskRunner.run(context, deadline);
let status: FullResult['status'] = 'passed';
if (context.dispatcher?.hasWorkerErrors() || context.rootSuite?.allTests().some(test => !test.ok()))
if (context.phases.find(p => p.dispatcher.hasWorkerErrors()) || context.rootSuite?.allTests().some(test => !test.ok()))
status = 'failed';
if (status === 'passed' && taskStatus !== 'passed')
status = taskStatus;

121
packages/playwright-test/src/runner/tasks.ts
View file

@ -17,16 +17,16 @@
import fs from 'fs';
import path from 'path';
import { promisify } from 'util';
import { rimraf } from 'playwright-core/lib/utilsBundle';
import { debug, rimraf } from 'playwright-core/lib/utilsBundle';
import { Dispatcher } from './dispatcher';
import type { TestRunnerPlugin, TestRunnerPluginRegistration } from '../plugins';
import type { Multiplexer } from '../reporters/multiplexer';
import type { TestGroup } from '../runner/testGroups';
import { createTestGroups, filterForShard } from '../runner/testGroups';
import { createTestGroups } from '../runner/testGroups';
import type { Task } from './taskRunner';
import { TaskRunner } from './taskRunner';
import type { Suite } from '../common/test';
import type { FullConfigInternal } from '../common/types';
import type { FullConfigInternal, FullProjectInternal } from '../common/types';
import { loadAllTests, loadGlobalHook } from './loadUtils';
import type { Matcher, TestFileFilter } from '../util';
@ -41,14 +41,22 @@ type TaskRunnerOptions = {
passWithNoTests?: boolean;
};
type ProjectWithTestGroups = {
project: FullProjectInternal;
projectSuite: Suite;
testGroups: TestGroup[];
};
export type TaskRunnerState = {
options: TaskRunnerOptions;
reporter: Multiplexer;
config: FullConfigInternal;
plugins: TestRunnerPlugin[];
testGroups: TestGroup[];
rootSuite?: Suite;
dispatcher?: Dispatcher;
phases: {
dispatcher: Dispatcher,
projects: ProjectWithTestGroups[]
}[];
};
export function createTaskRunner(config: FullConfigInternal, reporter: Multiplexer): TaskRunner<TaskRunnerState> {
@ -71,8 +79,7 @@ export function createTaskRunner(config: FullConfigInternal, reporter: Multiplex
return () => reporter.onEnd();
});
taskRunner.addTask('setup workers', createSetupWorkersTask());
taskRunner.addTask('test suite', async ({ dispatcher, testGroups }) => dispatcher!.run(testGroups));
taskRunner.addTask('test suite', createRunTestsTask());
return taskRunner;
}
@ -113,17 +120,6 @@ function createGlobalSetupTask(): Task<TaskRunnerState> {
};
}
function createSetupWorkersTask(): Task<TaskRunnerState> {
return async params => {
const { config, reporter } = params;
const dispatcher = new Dispatcher(config, reporter);
params.dispatcher = dispatcher;
return async () => {
await dispatcher.stop();
};
};
}
function createRemoveOutputDirsTask(): Task<TaskRunnerState> {
return async ({ config, options }) => {
const outputDirs = new Set<string>();
@ -151,20 +147,95 @@ function createLoadTask(): Task<TaskRunnerState> {
const { config, reporter, options } = context;
context.rootSuite = await loadAllTests(config, reporter, options, errors);
// Fail when no tests.
if (!context.rootSuite.allTests().length && !context.options.passWithNoTests)
if (!context.rootSuite.allTests().length && !context.options.passWithNoTests && !config.shard)
throw new Error(`No tests found`);
};
}
function createTestGroupsTask(): Task<TaskRunnerState> {
return async context => {
const { config, rootSuite } = context;
const { config, rootSuite, reporter } = context;
for (const phase of buildPhases(rootSuite!.suites)) {
// Go over the phases, for each phase create list of task groups.
const projects: ProjectWithTestGroups[] = [];
for (const projectSuite of phase) {
const testGroups = createTestGroups(projectSuite, config.workers);
projects.push({
project: projectSuite._projectConfig!,
projectSuite,
testGroups,
});
}
for (const projectSuite of rootSuite!.suites)
context.testGroups.push(...createTestGroups(projectSuite, config.workers));
const testGroupsInPhase = projects.reduce((acc, project) => acc + project.testGroups.length, 0);
debug('pw:test:task')(`running phase with ${projects.map(p => p.project.name).sort()} projects, ${testGroupsInPhase} testGroups`);
context.phases.push({ dispatcher: new Dispatcher(config, reporter), projects });
context.config._maxConcurrentTestGroups = Math.max(context.config._maxConcurrentTestGroups, testGroupsInPhase);
}
if (context.config.shard)
filterForShard(context.config.shard, rootSuite!, context.testGroups);
context.config._maxConcurrentTestGroups = context.testGroups.length;
return async () => {
for (const { dispatcher } of context.phases.reverse())
await dispatcher.stop();
};
};
}
function createRunTestsTask(): Task<TaskRunnerState> {
return async context => {
const { phases } = context;
const successfulProjects = new Set<FullProjectInternal>();
for (const { dispatcher, projects } of phases) {
// Each phase contains dispatcher and a set of test groups.
// We don't want to run the test groups beloning to the projects
// that depend on the projects that failed previously.
const phaseTestGroups: TestGroup[] = [];
for (const { project, testGroups } of projects) {
const hasFailedDeps = project._depProjects.some(p => !successfulProjects.has(p));
if (!hasFailedDeps) {
phaseTestGroups.push(...testGroups);
} else {
for (const testGroup of testGroups) {
for (const test of testGroup.tests)
test._appendTestResult().status = 'skipped';
}
}
}
if (phaseTestGroups.length) {
await dispatcher!.run(phaseTestGroups);
await dispatcher.stop();
}
// If the worker broke, fail everything, we have no way of knowing which
// projects failed.
if (!dispatcher.hasWorkerErrors()) {
for (const { project, projectSuite } of projects) {
const hasFailedDeps = project._depProjects.some(p => !successfulProjects.has(p));
if (!hasFailedDeps && !projectSuite.allTests().some(test => !test.ok()))
successfulProjects.add(project);
}
}
}
};
}
function buildPhases(projectSuites: Suite[]): Suite[][] {
const phases: Suite[][] = [];
const processed = new Set<FullProjectInternal>();
for (let i = 0; i < projectSuites.length; i++) {
const phase: Suite[] = [];
for (const projectSuite of projectSuites) {
if (processed.has(projectSuite._projectConfig!))
continue;
if (projectSuite._projectConfig!._depProjects.find(p => !processed.has(p)))
continue;
phase.push(projectSuite);
}
for (const projectSuite of phase)
processed.add(projectSuite._projectConfig!);
if (phase.length)
phases.push(phase);
}
return phases;
}

44
packages/playwright-test/src/runner/testGroups.ts
View file

@ -14,8 +14,8 @@
* limitations under the License.
*/
import { filterSuiteWithOnlySemantics } from '../common/suiteUtils';
import type { Suite, TestCase } from '../common/test';
import type { FullProjectInternal } from '../common/types';
export type TestGroup = {
workerHash: string;
@ -131,15 +131,10 @@ export function createTestGroups(projectSuite: Suite, workers: number): TestGrou
return result;
}
export async function filterForShard(shard: { total: number, current: number }, rootSuite: Suite, testGroups: TestGroup[]) {
// Each shard includes:
// - its portion of the regular tests
// - project setup tests for the projects that have regular tests in this shard
export function filterForShard(shard: { total: number, current: number }, filesByProject: Map<FullProjectInternal, string[]>): Map<FullProjectInternal, string[]> {
let shardableTotal = 0;
for (const group of testGroups)
shardableTotal += group.tests.length;
const shardTests = new Set<TestCase>();
for (const files of filesByProject.values())
shardableTotal += files.length;
// Each shard gets some tests.
const shardSize = Math.floor(shardableTotal / shard.total);
@ -150,27 +145,16 @@ export async function filterForShard(shard: { total: number, current: number },
const from = shardSize * currentShard + Math.min(extraOne, currentShard);
const to = from + shardSize + (currentShard < extraOne ? 1 : 0);
let current = 0;
const shardProjects = new Set<string>();
const shardTestGroups = [];
for (const group of testGroups) {
// Any test group goes to the shard that contains the first test of this group.
// So, this shard gets any group that starts at [from; to)
if (current >= from && current < to) {
shardProjects.add(group.projectId);
shardTestGroups.push(group);
for (const test of group.tests)
shardTests.add(test);
const result = new Map<FullProjectInternal, string[]>();
for (const [project, files] of filesByProject) {
const shardFiles: string[] = [];
for (const file of files) {
if (current >= from && current < to)
shardFiles.push(file);
++current;
}
current += group.tests.length;
}
testGroups.length = 0;
testGroups.push(...shardTestGroups);
if (!shardTests.size) {
// Filtering with "only semantics" does not work when we have zero tests - it leaves all the tests.
// We need an empty suite in this case.
rootSuite._entries = [];
} else {
filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test));
if (shardFiles.length)
result.set(project, shardFiles);
}
return result;
}

245
tests/playwright-test/deps.spec.ts Normal file
View file

@ -0,0 +1,245 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { test, expect, stripAnsi } from './playwright-test-fixtures';
test('should run projects with dependencies', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A' },
{ name: 'B', _deps: ['A'] },
{ name: 'C', _deps: ['A'] },
],
};`,
'test.spec.ts': `
const { test } = pwt;
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(extractLines(result.output)).toEqual(['A', 'B', 'C']);
});
test('should not run project if dependency failed', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A' },
{ name: 'B', _deps: ['A'] },
{ name: 'C', _deps: ['B'] },
],
};`,
'test.spec.ts': `
const { test } = pwt;
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
if (testInfo.project.name === 'B')
throw new Error('Failed project B');
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(1);
expect(result.failed).toBe(1);
expect(result.skipped).toBe(1);
expect(result.output).toContain('Failed project B');
expect(extractLines(result.output)).toEqual(['A', 'B']);
});
test('should not run project if dependency failed (2)', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'A1' },
{ name: 'A2', _deps: ['A1'] },
{ name: 'A3', _deps: ['A2'] },
{ name: 'B1' },
{ name: 'B2', _deps: ['B1'] },
{ name: 'B3', _deps: ['B2'] },
],
};`,
'test.spec.ts': `
const { test } = pwt;
test('test', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
if (testInfo.project.name === 'B1')
throw new Error('Failed project B1');
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(extractLines(result.output).sort()).toEqual(['A1', 'A2', 'A3', 'B1']);
});
test('should filter by project list, but run deps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'A' },
{ name: 'B' },
{ name: 'C', _deps: ['A'] },
{ name: 'D' },
] };
`,
'test.spec.ts': `
const { test } = pwt;
test('pass', async ({}, testInfo) => {
console.log('\\n%%' + testInfo.project.name);
});
`
}, { project: ['C', 'D'] });
expect(result.passed).toBe(3);
expect(result.failed).toBe(0);
expect(result.skipped).toBe(0);
expect(extractLines(result.output).sort()).toEqual(['A', 'C', 'D']);
});
test('should not filter dependency by file name', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'A' },
{ name: 'B', _deps: ['A'] },
] };
`,
'one.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
'two.spec.ts': `pwt.test('pass', () => { });`,
}, undefined, undefined, { additionalArgs: ['two.spec.ts'] });
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('1) [A] › one.spec.ts:4:5 › fails');
});
test('should not filter dependency by only', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'setup', testMatch: /setup.ts/ },
{ name: 'browser', _deps: ['setup'] },
] };
`,
'setup.ts': `
pwt.test('passes', () => {
console.log('\\n%% setup in ' + pwt.test.info().project.name);
});
pwt.test.only('passes 2', () => {
console.log('\\n%% setup 2 in ' + pwt.test.info().project.name);
});
`,
'test.spec.ts': `pwt.test('pass', () => {
console.log('\\n%% test in ' + pwt.test.info().project.name);
});`,
});
expect(result.exitCode).toBe(0);
expect(extractLines(result.output)).toEqual(['setup in setup', 'setup 2 in setup', 'test in browser']);
});
test('should not filter dependency by only 2', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'setup', testMatch: /setup.ts/ },
{ name: 'browser', _deps: ['setup'] },
] };
`,
'setup.ts': `
pwt.test('passes', () => {
console.log('\\n%% setup in ' + pwt.test.info().project.name);
});
pwt.test.only('passes 2', () => {
console.log('\\n%% setup 2 in ' + pwt.test.info().project.name);
});
`,
'test.spec.ts': `pwt.test('pass', () => {
console.log('\\n%% test in ' + pwt.test.info().project.name);
});`,
}, { project: ['setup'] });
expect(result.exitCode).toBe(0);
expect(extractLines(result.output)).toEqual(['setup 2 in setup']);
});
test('should not filter dependency by only 3', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'setup', testMatch: /setup.*.ts/ },
{ name: 'browser', _deps: ['setup'] },
] };
`,
'setup-1.ts': `
pwt.test('setup 1', () => {
console.log('\\n%% setup in ' + pwt.test.info().project.name);
});
`,
'setup-2.ts': `
pwt.test('setup 2', () => {
console.log('\\n%% setup 2 in ' + pwt.test.info().project.name);
});
`,
'test.spec.ts': `pwt.test('pass', () => {
console.log('\\n%% test in ' + pwt.test.info().project.name);
});`,
}, undefined, undefined, { additionalArgs: ['setup-2.ts'] });
expect(result.exitCode).toBe(0);
expect(extractLines(result.output)).toEqual(['setup 2 in setup']);
});
test('should report skipped dependent tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'setup', testMatch: /setup.ts/ },
{ name: 'browser', _deps: ['setup'] },
] };
`,
'setup.ts': `
pwt.test('setup', () => {
expect(1).toBe(2);
});
`,
'test.spec.ts': `pwt.test('pass', () => {});`,
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.skipped).toBe(1);
expect(result.results.length).toBe(2);
});
test('should report circular dependencies', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'A', _deps: ['B'] },
{ name: 'B', _deps: ['A'] },
] };
`,
'test.spec.ts': `pwt.test('pass', () => {});`,
});
expect(result.exitCode).toBe(1);
expect(result.output).toContain('Circular dependency detected between projects.');
});
function extractLines(output: string): string[] {
return stripAnsi(output).split('\n').filter(line => line.startsWith('%%')).map(line => line.substring(2).trim());
}

36
tests/playwright-test/global-scripts.spec.ts
View file

@ -16,8 +16,6 @@
import path from 'path';
import { test, expect } from './playwright-test-fixtures';
test.fixme(true, 'Restore this');
type Timeline = { titlePath: string[], event: 'begin' | 'end' }[];
function formatTimeline(timeline: Timeline) {
@ -70,11 +68,15 @@ test('should work for one project', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
globalScripts: /.*global.ts/,
projects: [
{
name: 'setup',
testMatch: /.*global.ts/,
},
{
name: 'p1',
testMatch: /.*.test.ts/,
_deps: ['setup'],
},
]
};`,
@ -92,10 +94,10 @@ test('should work for one project', async ({ runGroups }, testInfo) => {
const { exitCode, passed, timeline } = await runGroups(files);
expect(exitCode).toBe(0);
expect(passed).toBe(4);
expect(formatTimeline(timeline)).toEqual(`Global Scripts > global.ts > setup1 [begin]
Global Scripts > global.ts > setup1 [end]
Global Scripts > global.ts > setup2 [begin]
Global Scripts > global.ts > setup2 [end]
expect(formatTimeline(timeline)).toEqual(`setup > global.ts > setup1 [begin]
setup > global.ts > setup1 [end]
setup > global.ts > setup2 [begin]
setup > global.ts > setup2 [end]
p1 > a.test.ts > test1 [begin]
p1 > a.test.ts > test1 [end]
p1 > a.test.ts > test2 [begin]
@ -106,15 +108,20 @@ test('should work for several projects', async ({ runGroups }, testInfo) => {
const files = {
'playwright.config.ts': `
module.exports = {
globalScripts: /.*global.ts/,
projects: [
{
name: 'setup',
testMatch: /.*global.ts/,
},
{
name: 'p1',
testMatch: /.*a.test.ts/,
_deps: ['setup'],
},
{
name: 'p2',
testMatch: /.*b.test.ts/,
_deps: ['setup'],
},
]
};`,
@ -144,15 +151,20 @@ test('should skip tests if global setup fails', async ({ runGroups }, testInfo)
const files = {
'playwright.config.ts': `
module.exports = {
globalScripts: /.*global.ts/,
projects: [
{
name: 'setup',
testMatch: /.*global.ts/,
},
{
name: 'p1',
testMatch: /.*a.test.ts/,
_deps: ['setup'],
},
{
name: 'p2',
testMatch: /.*b.test.ts/,
_deps: ['setup'],
},
]
};`,
@ -181,10 +193,14 @@ test('should run setup in each project shard', async ({ runGroups }, testInfo) =
const files = {
'playwright.config.ts': `
module.exports = {
globalScripts: /.*global.ts/,
projects: [
{
name: 'setup',
testMatch: /.*global.ts/,
},
{
name: 'p1',
_deps: ['setup'],
},
]
};`,

7
tests/playwright-test/global-setup.spec.ts
View file

@ -24,10 +24,7 @@ test('globalSetup and globalTeardown should work', async ({ runInlineTest }) =>
testDir: '..',
globalSetup: './globalSetup',
globalTeardown: path.join(__dirname, 'globalTeardown.ts'),
projects: [
{ name: 'p1' },
{ name: 'p2' },
]
projects: [{ name: 'p1' }]
};
`,
'dir/globalSetup.ts': `
@ -46,7 +43,7 @@ test('globalSetup and globalTeardown should work', async ({ runInlineTest }) =>
console.log('\\n%%from-test');
});
`,
}, { 'project': 'p2', 'config': 'dir' });
}, { 'config': 'dir' });
expect(result.passed).toBe(1);
expect(result.failed).toBe(0);
expect(stripAnsi(result.output).split('\n').filter(line => line.startsWith('%%'))).toEqual([

19
tests/playwright-test/shard.spec.ts
View file

@ -21,23 +21,32 @@ const tests = {
export const headlessTest = pwt.test.extend({ headless: false });
export const headedTest = pwt.test.extend({ headless: true });
`,
'a.spec.ts': `
'a1.spec.ts': `
import { headlessTest, headedTest } from './helper';
headlessTest('test1', async () => {
console.log('test1-done');
});
`,
'a2.spec.ts': `
import { headlessTest, headedTest } from './helper';
headedTest('test2', async () => {
console.log('test2-done');
});
`,
'a3.spec.ts': `
import { headlessTest, headedTest } from './helper';
headlessTest('test3', async () => {
console.log('test3-done');
});
`,
'b.spec.ts': `
'b1.spec.ts': `
import { headlessTest, headedTest } from './helper';
headlessTest('test4', async () => {
console.log('test4-done');
});
`,
'b2.spec.ts': `
import { headlessTest, headedTest } from './helper';
headedTest('test5', async () => {
console.log('test5-done');
});
@ -50,8 +59,8 @@ test('should respect shard=1/2', async ({ runInlineTest }) => {
expect(result.passed).toBe(3);
expect(result.skipped).toBe(0);
expect(result.output).toContain('test1-done');
expect(result.output).toContain('test2-done');
expect(result.output).toContain('test3-done');
expect(result.output).toContain('test4-done');
});
test('should respect shard=2/2', async ({ runInlineTest }) => {
@ -59,7 +68,7 @@ test('should respect shard=2/2', async ({ runInlineTest }) => {
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.skipped).toBe(0);
expect(result.output).toContain('test2-done');
expect(result.output).toContain('test4-done');
expect(result.output).toContain('test5-done');
});
@ -83,6 +92,6 @@ test('should respect shard=1/2 in config', async ({ runInlineTest }) => {
expect(result.passed).toBe(3);
expect(result.skipped).toBe(0);
expect(result.output).toContain('test1-done');
expect(result.output).toContain('test2-done');
expect(result.output).toContain('test3-done');
expect(result.output).toContain('test4-done');
});