diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index 0ad4b68294..8c491514ae 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -55,7 +55,7 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}-* - run: node tests/config/checkCoverage.js ${{ matrix.browser }} - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() @@ -87,7 +87,7 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chromium-tip-of-tree - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium-* env: PWTEST_CHANNEL: chromium-tip-of-tree PWTEST_BOT_NAME: "${{ matrix.os }}-chromium-tip-of-tree" diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 816bf5c3f9..7659c338bc 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -42,7 +42,7 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}-* - run: node tests/config/checkCoverage.js ${{ matrix.browser }} - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() @@ -75,7 +75,7 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - - run: npm run test -- --project=${{ matrix.browser }} + - run: npm run test -- --project=${{ matrix.browser }}-* - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash @@ -106,10 +106,10 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - - run: npm run test -- --project=${{ matrix.browser }} --workers=1 + - run: npm run test -- --project=${{ matrix.browser }}-* --workers=1 if: matrix.browser == 'firefox' shell: bash - - run: npm run test -- --project=${{ matrix.browser }} + - run: npm run test -- --project=${{ matrix.browser }}-* if: matrix.browser != 'firefox' shell: bash - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json @@ -175,9 +175,9 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} --headed + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}-* --headed if: always() && startsWith(matrix.os, 'ubuntu-') - - run: npm run test -- --project=${{ matrix.browser }} --headed + - run: npm run test -- --project=${{ matrix.browser }}-* --headed if: always() && !startsWith(matrix.os, 'ubuntu-') - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() @@ -247,7 +247,7 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium ${{ matrix.channel }} - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}-* env: PWTEST_TRACE: 1 PWTEST_CHANNEL: ${{ matrix.channel }} @@ -868,7 +868,7 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps chromium - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=chromium-* env: PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW: 1 - run: node tests/config/checkCoverage.js chromium diff --git a/.github/workflows/tests_service.yml b/.github/workflows/tests_service.yml index 9c932f38e4..b8c192f988 100644 --- a/.github/workflows/tests_service.yml +++ b/.github/workflows/tests_service.yml @@ -23,7 +23,7 @@ jobs: env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} --workers=10 --retries=0 + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}-* --workers=10 --retries=0 env: PWTEST_MODE: service2 PWTEST_TRACE: 1 diff --git a/.github/workflows/tests_video.yml b/.github/workflows/tests_video.yml index f39b544f93..3acda7cec7 100644 --- a/.github/workflows/tests_video.yml +++ b/.github/workflows/tests_video.yml @@ -32,7 +32,7 @@ jobs: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}-* env: PWTEST_VIDEO: 1 - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json diff --git a/package.json b/package.json index fdbafad27c..253206f667 100644 --- a/package.json +++ b/package.json @@ -16,9 +16,9 @@ }, "license": "Apache-2.0", "scripts": { - "ctest": "playwright test --config=tests/library/playwright.config.ts --project=chromium", - "ftest": "playwright test --config=tests/library/playwright.config.ts --project=firefox", - "wtest": "playwright test --config=tests/library/playwright.config.ts --project=webkit", + "ctest": "playwright test --config=tests/library/playwright.config.ts --project=chromium-*", + "ftest": "playwright test --config=tests/library/playwright.config.ts --project=firefox-*", + "wtest": "playwright test --config=tests/library/playwright.config.ts --project=webkit-*", "atest": "playwright test --config=tests/android/playwright.config.ts", "etest": "playwright test --config=tests/electron/playwright.config.ts", "webview2test": "playwright test --config=tests/webview2/playwright.config.ts", diff --git a/packages/html-reporter/src/filter.ts b/packages/html-reporter/src/filter.ts index 6f7ef14525..97cf675936 100644 --- a/packages/html-reporter/src/filter.ts +++ b/packages/html-reporter/src/filter.ts @@ -14,7 +14,6 @@ limitations under the License. */ -import { testCaseLabels } from './labelUtils'; import type { TestCaseSummary } from './types'; export class Filter { @@ -108,13 +107,13 @@ export class Filter { if (test.outcome === 'skipped') status = 'skipped'; const searchValues: SearchValues = { - text: (status + ' ' + test.projectName + ' ' + (test.botName || '') + ' ' + test.location.file + ' ' + test.path.join(' ') + ' ' + test.title).toLowerCase(), + text: (status + ' ' + test.projectName + ' ' + test.tags.join(' ') + ' ' + test.location.file + ' ' + test.path.join(' ') + ' ' + test.title).toLowerCase(), project: test.projectName.toLowerCase(), status: status as any, file: test.location.file, line: String(test.location.line), column: String(test.location.column), - labels: testCaseLabels(test).map(label => label.toLowerCase()), + labels: test.tags.map(tag => tag.toLowerCase()), }; (test as any).searchValues = searchValues; } diff --git a/packages/html-reporter/src/labelUtils.tsx b/packages/html-reporter/src/labelUtils.tsx index 57d9c43c9a..014ec77d59 100644 --- a/packages/html-reporter/src/labelUtils.tsx +++ b/packages/html-reporter/src/labelUtils.tsx @@ -14,22 +14,6 @@ * limitations under the License. */ -import type { TestCaseSummary } from './types'; - -const labelsSymbol = Symbol('labels'); - -// Note: all labels start with "@" -export function testCaseLabels(test: TestCaseSummary): string[] { - if (!(test as any)[labelsSymbol]) { - const labels: string[] = []; - if (test.botName) - labels.push('@' + test.botName); - labels.push(...test.tags); - (test as any)[labelsSymbol] = labels; - } - return (test as any)[labelsSymbol]; -} - // hash string to integer in range [0, 6] for color index, to get same color for same tag export function hashStringToInt(str: string) { let hash = 0; diff --git a/packages/html-reporter/src/testCaseView.tsx b/packages/html-reporter/src/testCaseView.tsx index 4b79908edf..eecbdac9f5 100644 --- a/packages/html-reporter/src/testCaseView.tsx +++ b/packages/html-reporter/src/testCaseView.tsx @@ -23,7 +23,7 @@ import { ProjectLink } from './links'; import { statusIcon } from './statusIcon'; import './testCaseView.css'; import { TestResultView } from './testResultView'; -import { hashStringToInt, testCaseLabels } from './labelUtils'; +import { hashStringToInt } from './labelUtils'; import { msToString } from './uiUtils'; export const TestCaseView: React.FC<{ @@ -37,7 +37,7 @@ export const TestCaseView: React.FC<{ const labels = React.useMemo(() => { if (!test) return undefined; - return testCaseLabels(test); + return test.tags; }, [test]); return
diff --git a/packages/html-reporter/src/testFileView.tsx b/packages/html-reporter/src/testFileView.tsx index 619d6263d2..4f508c42b8 100644 --- a/packages/html-reporter/src/testFileView.tsx +++ b/packages/html-reporter/src/testFileView.tsx @@ -23,7 +23,7 @@ import { generateTraceUrl, Link, navigate, ProjectLink } from './links'; import { statusIcon } from './statusIcon'; import './testFileView.css'; import { video, image, trace } from './icons'; -import { hashStringToInt, testCaseLabels } from './labelUtils'; +import { hashStringToInt } from './labelUtils'; export const TestFileView: React.FC {report.projectNames.length > 1 && !!test.projectName && } - +
{msToString(test.duration)} diff --git a/packages/html-reporter/src/types.ts b/packages/html-reporter/src/types.ts index d7ada9cbe6..733e88e8b9 100644 --- a/packages/html-reporter/src/types.ts +++ b/packages/html-reporter/src/types.ts @@ -66,7 +66,6 @@ export type TestCaseSummary = { title: string; path: string[]; projectName: string; - botName?: string; location: Location; annotations: TestCaseAnnotation[]; tags: string[]; diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index cfe294a75c..1049bcf5b7 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -132,7 +132,7 @@ export class FullConfigInternal { const projectConfigs = configCLIOverrides.projects || userConfig.projects || [userConfig]; this.projects = projectConfigs.map(p => new FullProjectInternal(configDir, userConfig, this, p, this.configCLIOverrides, throwawayArtifactsPath)); resolveProjectDependencies(this.projects); - this._assignUniqueProjectIds(this.projects); + this._assignMissingProjectNames(this.projects); setTransformConfig({ babelPlugins: privateConfiguration?.babelPlugins || [], external: userConfig.build?.external || [], @@ -140,19 +140,21 @@ export class FullConfigInternal { this.config.projects = this.projects.map(p => p.project); } - private _assignUniqueProjectIds(projects: FullProjectInternal[]) { + private _assignMissingProjectNames(projects: FullProjectInternal[]) { + const userProvidedNames = new Set(projects.map(p => p.project.name)); const usedNames = new Set(); + let counter = 0; for (const p of projects) { - const name = p.project.name || ''; - for (let i = 0; i < projects.length; ++i) { - const candidate = name + (i ? i : ''); - if (usedNames.has(candidate)) - continue; - p.id = candidate; - (p.project as any).__projectId = p.id; - usedNames.add(candidate); - break; + if (p.project.name && usedNames.has(p.project.name)) + throw new Error(`Duplicate project name "${p.project.name}" in ${this.config.configFile}`); + + if (!p.project.name && usedNames.has('')) { + let candidate = 'project_' + String(++counter); + while (userProvidedNames.has(candidate)) + candidate = 'project_' + String(++counter); + p.project.name = candidate; } + usedNames.add(p.project.name); } } } @@ -164,7 +166,6 @@ export class FullProjectInternal { readonly expect: Project['expect']; readonly respectGitIgnore: boolean; readonly snapshotPathTemplate: string; - id = ''; deps: FullProjectInternal[] = []; teardown: FullProjectInternal | undefined; @@ -285,7 +286,3 @@ export const defaultGrep = /.*/; export const defaultReporter = process.env.CI ? 'dot' : 'list'; const configInternalSymbol = Symbol('configInternalSymbol'); - -export function getProjectId(project: FullProject): string { - return (project as any).__projectId!; -} \ No newline at end of file diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index bb35f6c06d..02b56df172 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -62,7 +62,7 @@ export type WorkerInitParams = { workerIndex: number; parallelIndex: number; repeatEachIndex: number; - projectId: string; + projectName: string; config: SerializedConfig; artifactsDir: string; }; diff --git a/packages/playwright/src/common/poolBuilder.ts b/packages/playwright/src/common/poolBuilder.ts index cc1bc095c9..8703df348d 100644 --- a/packages/playwright/src/common/poolBuilder.ts +++ b/packages/playwright/src/common/poolBuilder.ts @@ -75,7 +75,7 @@ export class PoolBuilder { if (!this._testTypePools.has(testType)) { const optionOverrides = { overrides: this._project?.project?.use ?? {}, - location: { file: `project#${this._project?.id}`, line: 1, column: 1 } + location: { file: `project#${this._project?.project.name}`, line: 1, column: 1 } }; const pool = new FixturePool(testType.fixtures, e => this._handleLoadError(e, testErrors), undefined, undefined, optionOverrides); this._testTypePools.set(testType, pool); diff --git a/packages/playwright/src/common/suiteUtils.ts b/packages/playwright/src/common/suiteUtils.ts index 81dc9d5465..46b0d0aa18 100644 --- a/packages/playwright/src/common/suiteUtils.ts +++ b/packages/playwright/src/common/suiteUtils.ts @@ -53,10 +53,9 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit suite._fileId = fileId; // At the point of the query, suite is not yet attached to the project, so we only get file, describe and test titles. const [file, ...titles] = test.titlePath(); - const testIdExpression = `[project=${project.id}]${toPosixPath(file)}\x1e${titles.join('\x1e')}`; + const testIdExpression = `[project=${project.project.name}]${toPosixPath(file)}\x1e${titles.join('\x1e')}`; const testId = fileId + '-' + calculateSha1(testIdExpression).slice(0, 20); test.id = testId; - test._projectId = project.id; // Inherit properties from parent suites. let inheritedRetries: number | undefined; @@ -80,7 +79,7 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit // We only compute / set digest in the runner. if (test._poolDigest) - test._workerHash = `${project.id}-${test._poolDigest}-0`; + test._workerHash = `${project.project.name}-${test._poolDigest}-0`; }); return result; @@ -91,13 +90,13 @@ export function applyRepeatEachIndex(project: FullProjectInternal, fileSuite: Su fileSuite.forEachTest((test, suite) => { if (repeatEachIndex) { const [file, ...titles] = test.titlePath(); - const testIdExpression = `[project=${project.id}]${toPosixPath(file)}\x1e${titles.join('\x1e')} (repeat:${repeatEachIndex})`; + const testIdExpression = `[project=${project.project.name}]${toPosixPath(file)}\x1e${titles.join('\x1e')} (repeat:${repeatEachIndex})`; const testId = suite._fileId + '-' + calculateSha1(testIdExpression).slice(0, 20); test.id = testId; test.repeatEachIndex = repeatEachIndex; if (test._poolDigest) - test._workerHash = `${project.id}-${test._poolDigest}-${repeatEachIndex}`; + test._workerHash = `${project.project.name}-${test._poolDigest}-${repeatEachIndex}`; } }); } diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 0d05ac5dfd..290025cf5c 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -252,7 +252,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { _pool: FixturePool | undefined; _poolDigest = ''; _workerHash = ''; - _projectId = ''; // Annotations known statically before running the test, e.g. `test.skip()` or `test(title, { annotation }, body)`. _staticAnnotations: Annotation[] = []; // Explicitly declared tags that are not a part of the title. @@ -314,7 +313,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { staticAnnotations: this._staticAnnotations.slice(), annotations: this.annotations.slice(), tags: this._tags.slice(), - projectId: this._projectId, }; } @@ -331,7 +329,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { test._staticAnnotations = data.staticAnnotations; test.annotations = data.annotations; test._tags = data.tags; - test._projectId = data.projectId; return test; } diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index a7f9f412f1..3ab54e7657 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -36,7 +36,6 @@ export type JsonPattern = { }; export type JsonProject = { - id: string; grep: JsonPattern[]; grepInvert: JsonPattern[]; metadata: Metadata; @@ -201,7 +200,7 @@ export class TeleReporterReceiver { } private _onProject(project: JsonProject) { - let projectSuite = this._rootSuite.suites.find(suite => suite.project()!.__projectId === project.id); + let projectSuite = this._rootSuite.suites.find(suite => suite.project()!.name === project.name); if (!projectSuite) { projectSuite = new TeleSuite(project.name, 'project'); this._rootSuite.suites.push(projectSuite); @@ -331,7 +330,6 @@ export class TeleReporterReceiver { private _parseProject(project: JsonProject): TeleFullProject { return { - __projectId: project.id, metadata: project.metadata, name: project.name, outputDir: this._absolutePath(project.outputDir), @@ -593,7 +591,7 @@ class TeleTestResult implements reporterTypes.TestResult { } } -export type TeleFullProject = FullProject & { __projectId: string }; +export type TeleFullProject = FullProject; export const baseFullConfig: reporterTypes.FullConfig = { forbidOnly: false, diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 8c0adcd5c0..1b66da35f7 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -241,7 +241,7 @@ class HtmlBuilder { } const { testFile, testFileSummary } = fileEntry; const testEntries: TestEntry[] = []; - this._processJsonSuite(fileSuite, fileId, projectSuite.project()!.name, projectSuite.project()!.metadata?.reportName, [], testEntries); + this._processJsonSuite(fileSuite, fileId, projectSuite.project()!.name, [], testEntries); for (const test of testEntries) { testFile.tests.push(test.testCase); testFileSummary.tests.push(test.testCaseSummary); @@ -341,13 +341,13 @@ class HtmlBuilder { this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName); } - private _processJsonSuite(suite: Suite, fileId: string, projectName: string, botName: string | undefined, path: string[], outTests: TestEntry[]) { + private _processJsonSuite(suite: Suite, fileId: string, projectName: string, path: string[], outTests: TestEntry[]) { const newPath = [...path, suite.title]; - suite.suites.forEach(s => this._processJsonSuite(s, fileId, projectName, botName, newPath, outTests)); - suite.tests.forEach(t => outTests.push(this._createTestEntry(fileId, t, projectName, botName, newPath))); + suite.suites.forEach(s => this._processJsonSuite(s, fileId, projectName, newPath, outTests)); + suite.tests.forEach(t => outTests.push(this._createTestEntry(fileId, t, projectName, newPath))); } - private _createTestEntry(fileId: string, test: TestCasePublic, projectName: string, botName: string | undefined, path: string[]): TestEntry { + private _createTestEntry(fileId: string, test: TestCasePublic, projectName: string, path: string[]): TestEntry { const duration = test.results.reduce((a, r) => a + r.duration, 0); const location = this._relativeLocation(test.location)!; path = path.slice(1); @@ -363,7 +363,6 @@ class HtmlBuilder { testId, title: test.title, projectName, - botName, location, duration, // Annotations can be pushed directly, with a wrong type. @@ -378,7 +377,6 @@ class HtmlBuilder { testId, title: test.title, projectName, - botName, location, duration, // Annotations can be pushed directly, with a wrong type. diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts index 36ba678258..9520ffc473 100644 --- a/packages/playwright/src/reporters/json.ts +++ b/packages/playwright/src/reporters/json.ts @@ -19,7 +19,6 @@ import path from 'path'; import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep, JSONReportError } from '../../types/testReporter'; import { formatError, prepareErrorStack } from './base'; import { MultiMap, assert, toPosixPath } from 'playwright-core/lib/utils'; -import { getProjectId } from '../common/config'; import EmptyReporter from './empty'; class JSONReporter extends EmptyReporter { @@ -64,7 +63,7 @@ class JSONReporter extends EmptyReporter { repeatEach: project.repeatEach, retries: project.retries, metadata: project.metadata, - id: getProjectId(project), + id: project.name, name: project.name, testDir: toPosixPath(project.testDir), testIgnore: serializePatterns(project.testIgnore), @@ -92,11 +91,10 @@ class JSONReporter extends EmptyReporter { private _mergeSuites(suites: Suite[]): JSONReportSuite[] { const fileSuites = new MultiMap(); for (const projectSuite of suites) { - const projectId = getProjectId(projectSuite.project()!); const projectName = projectSuite.project()!.name; for (const fileSuite of projectSuite.suites) { const file = fileSuite.location!.file; - const serialized = this._serializeSuite(projectId, projectName, fileSuite); + const serialized = this._serializeSuite(projectName, fileSuite); if (serialized) fileSuites.set(file, serialized); } @@ -153,35 +151,35 @@ class JSONReporter extends EmptyReporter { } } - private _serializeSuite(projectId: string, projectName: string, suite: Suite): null | JSONReportSuite { + private _serializeSuite(projectName: string, suite: Suite): null | JSONReportSuite { if (!suite.allTests().length) return null; - const suites = suite.suites.map(suite => this._serializeSuite(projectId, projectName, suite)).filter(s => s) as JSONReportSuite[]; + const suites = suite.suites.map(suite => this._serializeSuite(projectName, suite)).filter(s => s) as JSONReportSuite[]; return { title: suite.title, ...this._relativeLocation(suite.location), - specs: suite.tests.map(test => this._serializeTestSpec(projectId, projectName, test)), + specs: suite.tests.map(test => this._serializeTestSpec(projectName, test)), suites: suites.length ? suites : undefined, }; } - private _serializeTestSpec(projectId: string, projectName: string, test: TestCase): JSONReportSpec { + private _serializeTestSpec(projectName: string, test: TestCase): JSONReportSpec { return { title: test.title, ok: test.ok(), tags: test.tags.map(tag => tag.substring(1)), // Strip '@'. - tests: [this._serializeTest(projectId, projectName, test)], + tests: [this._serializeTest(projectName, test)], id: test.id, ...this._relativeLocation(test.location), }; } - private _serializeTest(projectId: string, projectName: string, test: TestCase): JSONReportTest { + private _serializeTest(projectName: string, test: TestCase): JSONReportTest { return { timeout: test.timeout, annotations: test.annotations, expectedStatus: test.expectedStatus, - projectId, + projectId: projectName, projectName, results: test.results.map(r => this._serializeTestResult(r, test)), status: test.outcome(), diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts index 096cc66c50..bc7eeb02a4 100644 --- a/packages/playwright/src/reporters/merge.ts +++ b/packages/playwright/src/reporters/merge.ts @@ -23,7 +23,7 @@ import { TeleReporterReceiver } from '../isomorphic/teleReceiver'; import { JsonStringInternalizer, StringInternPool } from '../isomorphic/stringInternPool'; import { createReporters } from '../runner/reporters'; import { Multiplexer } from './multiplexer'; -import { ZipFile, calculateSha1 } from 'playwright-core/lib/utils'; +import { ZipFile } from 'playwright-core/lib/utils'; import { currentBlobReportVersion, type BlobReportMetadata } from './blob'; import { relativeFilePath } from '../util'; import type { TestError } from '../../types/testReporter'; @@ -183,22 +183,15 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool: return a.file.localeCompare(b.file); }); - const saltSet = new Set(); - printStatus(`merging events`); const reports: ReportData[] = []; - for (const { file, parsedEvents, metadata, localPath } of blobs) { + for (let i = 0; i < blobs.length; ++i) { // Generate unique salt for each blob. - const sha1 = calculateSha1(metadata.name || path.basename(file)).substring(0, 16); - let salt = sha1; - for (let i = 0; saltSet.has(salt); i++) - salt = sha1 + '-' + i; - saltSet.add(salt); - + const { parsedEvents, metadata, localPath } = blobs[i]; const eventPatchers = new JsonEventPatchers(); - eventPatchers.patchers.push(new IdsPatcher(stringPool, metadata.name, salt)); + eventPatchers.patchers.push(new IdsPatcher(stringPool, metadata.name, String(i))); // Only patch path separators if we are merging reports with explicit config. if (rootDirOverride) eventPatchers.patchers.push(new PathSeparatorPatcher(metadata.pathSeparator)); @@ -380,13 +373,17 @@ class IdsPatcher { private _onProject(project: JsonProject) { project.metadata = project.metadata ?? {}; - project.metadata.reportName = this._reportName; - project.id = this._stringPool.internString(project.id + this._salt); project.suites.forEach(suite => this._updateTestIds(suite)); } private _updateTestIds(suite: JsonSuite) { - suite.tests.forEach(test => test.testId = this._mapTestId(test.testId)); + suite.tests.forEach(test => { + test.testId = this._mapTestId(test.testId); + if (this._reportName) { + test.tags = test.tags || []; + test.tags.unshift('@' + this._reportName); + } + }); suite.suites.forEach(suite => this._updateTestIds(suite)); } diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 44a9908300..96178bedf8 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -17,7 +17,6 @@ import path from 'path'; import { createGuid } from 'playwright-core/lib/utils'; import type { FullConfig, FullResult, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; -import { getProjectId } from '../common/config'; import type { JsonAttachment, JsonConfig, JsonEvent, JsonFullResult, JsonProject, JsonStdIOType, JsonSuite, JsonTestCase, JsonTestEnd, JsonTestResultEnd, JsonTestResultStart, JsonTestStepEnd, JsonTestStepStart } from '../isomorphic/teleReceiver'; import { serializeRegexPatterns } from '../isomorphic/teleReceiver'; import type { ReporterV2 } from './reporterV2'; @@ -158,7 +157,6 @@ export class TeleReporterEmitter implements ReporterV2 { private _serializeProject(suite: Suite): JsonProject { const project = suite.project()!; const report: JsonProject = { - id: getProjectId(project), metadata: project.metadata, name: project.name, outputDir: this._relativePath(project.outputDir), diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index da2b759aa6..a8cd15a3b6 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -180,9 +180,9 @@ export class Dispatcher { } _createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: SerializedConfig) { - const projectConfig = this._config.projects.find(p => p.id === testGroup.projectId)!; + const projectConfig = this._config.projects.find(p => p.project.name === testGroup.projectName)!; const outputDir = projectConfig.project.outputDir; - const worker = new WorkerHost(testGroup, parallelIndex, loaderData, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir); + const worker = new WorkerHost(testGroup, parallelIndex, loaderData, this._extraEnvByProjectId.get(testGroup.projectName) || {}, outputDir); const handleOutput = (params: TestOutputPayload) => { const chunk = chunkFromParams(params); if (worker.didFail()) { @@ -212,8 +212,8 @@ export class Dispatcher { this._reporter.onError(error); }); worker.on('exit', () => { - const producedEnv = this._producedEnvByProjectId.get(testGroup.projectId) || {}; - this._producedEnvByProjectId.set(testGroup.projectId, { ...producedEnv, ...worker.producedEnv() }); + const producedEnv = this._producedEnvByProjectId.get(testGroup.projectName) || {}; + this._producedEnvByProjectId.set(testGroup.projectName, { ...producedEnv, ...worker.producedEnv() }); }); return worker; } diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 1cd57034e4..80bfd268ab 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -284,10 +284,10 @@ function createRunTestsTask(): Task { // Inherit extra environment variables from dependencies. let extraEnv: Record = {}; for (const dep of project.deps) - extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(dep.id) }; + extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(dep.project.name) }; for (const setup of teardownToSetups.get(project) || []) - extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(setup.id) }; - extraEnvByProjectId.set(project.id, extraEnv); + extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(setup.project.name) }; + extraEnvByProjectId.set(project.project.name, extraEnv); const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p)); if (!hasFailedDeps) diff --git a/packages/playwright/src/runner/testGroups.ts b/packages/playwright/src/runner/testGroups.ts index 5a70bc84cc..6abc9d4fe4 100644 --- a/packages/playwright/src/runner/testGroups.ts +++ b/packages/playwright/src/runner/testGroups.ts @@ -20,7 +20,7 @@ export type TestGroup = { workerHash: string; requireFile: string; repeatEachIndex: number; - projectId: string; + projectName: string; tests: TestCase[]; }; @@ -56,7 +56,7 @@ export function createTestGroups(projectSuite: Suite, workers: number): TestGrou workerHash: test._workerHash, requireFile: test._requireFile, repeatEachIndex: test.repeatEachIndex, - projectId: test._projectId, + projectName: projectSuite.project()!.name, tests: [], }; }; diff --git a/packages/playwright/src/runner/workerHost.ts b/packages/playwright/src/runner/workerHost.ts index c9029aaff0..7ba6a9346a 100644 --- a/packages/playwright/src/runner/workerHost.ts +++ b/packages/playwright/src/runner/workerHost.ts @@ -47,7 +47,7 @@ export class WorkerHost extends ProcessHost { workerIndex: this.workerIndex, parallelIndex, repeatEachIndex: testGroup.repeatEachIndex, - projectId: testGroup.projectId, + projectName: testGroup.projectName, config, artifactsDir: path.join(outputDir, artifactsFolderName(workerIndex)) }; diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index e79bd61529..8e4c32e204 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -167,8 +167,8 @@ export class TestInfoImpl implements TestInfo { const fullTitleWithoutSpec = this.titlePath.slice(1).join(' '); let testOutputDir = trimLongString(sanitizedRelativePath + '-' + sanitizeForFilePath(fullTitleWithoutSpec)); - if (projectInternal.id) - testOutputDir += '-' + sanitizeForFilePath(projectInternal.id); + if (projectInternal.project.name) + testOutputDir += '-' + sanitizeForFilePath(projectInternal.project.name); if (this.retry) testOutputDir += '-retry' + this.retry; if (this.repeatEachIndex) diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index d678e485aa..86c1dee88b 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -206,7 +206,7 @@ export class WorkerMain extends ProcessRunner { return; this._config = await deserializeConfig(this._params.config); - this._project = this._config.projects.find(p => p.id === this._params.projectId)!; + this._project = this._config.projects.find(p => p.project.name === this._params.projectName)!; this._poolBuilder = PoolBuilder.createForWorker(this._project); } diff --git a/tests/android/playwright.config.ts b/tests/android/playwright.config.ts index 9b41c9b755..0c0c7fbd3a 100644 --- a/tests/android/playwright.config.ts +++ b/tests/android/playwright.config.ts @@ -50,23 +50,23 @@ const metadata = { }; config.projects!.push({ - name: 'android', + name: 'android-native', use: { loopback: '10.0.2.2', browserName: 'chromium', }, - snapshotPathTemplate: '{testDir}/{testFileDir}/{testFileName}-snapshots/{arg}{-projectName}{ext}', + snapshotPathTemplate: '{testDir}/{testFileDir}/{testFileName}-snapshots/{arg}-android{ext}', testDir: path.join(testDir, 'android'), metadata, }); config.projects!.push({ - name: 'android', + name: 'android-page', use: { loopback: '10.0.2.2', browserName: 'chromium', }, - snapshotPathTemplate: '{testDir}/{testFileDir}/{testFileName}-snapshots/{arg}{-projectName}{ext}', + snapshotPathTemplate: '{testDir}/{testFileDir}/{testFileName}-snapshots/{arg}-android{ext}', testDir: path.join(testDir, 'page'), metadata, }); diff --git a/tests/electron/playwright.config.ts b/tests/electron/playwright.config.ts index 59931f33ec..4807a7f919 100644 --- a/tests/electron/playwright.config.ts +++ b/tests/electron/playwright.config.ts @@ -51,7 +51,7 @@ const metadata = { }; config.projects.push({ - name: 'electron', + name: 'electron-api', use: { browserName: 'chromium', coverageName: 'electron', @@ -61,7 +61,7 @@ config.projects.push({ }); config.projects.push({ - name: 'electron', + name: 'electron-page', // Share screenshots with chromium. snapshotPathTemplate: '{testDir}/{testFileDir}/{testFileName}-snapshots/{arg}-chromium{ext}', use: { diff --git a/tests/library/playwright.config.ts b/tests/library/playwright.config.ts index 771b8f2499..568bf6f8aa 100644 --- a/tests/library/playwright.config.ts +++ b/tests/library/playwright.config.ts @@ -109,10 +109,10 @@ for (const browserName of browserNames) { const testIgnore: RegExp[] = browserNames.filter(b => b !== browserName).map(b => new RegExp(b)); for (const folder of ['library', 'page']) { config.projects.push({ - name: browserName, + name: `${browserName}-${folder}`, testDir: path.join(testDir, folder), testIgnore, - snapshotPathTemplate: '{testDir}/{testFileDir}/{testFileName}-snapshots/{arg}{-projectName}{ext}', + snapshotPathTemplate: `{testDir}/{testFileDir}/{testFileName}-snapshots/{arg}-${browserName}{ext}`, use: { mode, browserName, diff --git a/tests/playwright-test/basic.spec.ts b/tests/playwright-test/basic.spec.ts index 4c4b28ceeb..0b436d0b5c 100644 --- a/tests/playwright-test/basic.spec.ts +++ b/tests/playwright-test/basic.spec.ts @@ -237,6 +237,19 @@ test('should focus test from one project', async ({ runInlineTest }) => { expect(exitCode).toBe(0); }); +test('should throw on dupe project name', async ({ runInlineTest }) => { + const { exitCode, output } = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'a' }, + { name: 'a' }, + ] }; + `, + }, { reporter: 'list,json' }); + expect(output).toContain('Duplicate project name'); + expect(exitCode).toBe(1); +}); + test('should work with default export', async ({ runInlineTest }) => { const result = await runInlineTest({ 'file.spec.ts': ` diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts index 38fef69915..935f5a92b6 100644 --- a/tests/playwright-test/config.spec.ts +++ b/tests/playwright-test/config.spec.ts @@ -350,29 +350,6 @@ test('should filter by project list, case-insensitive', async ({ runInlineTest } expect(new Set(outputLines)).toEqual(new Set(['suite3', 'suite2'])); }); -test('should filter when duplicate project names exist', async ({ runInlineTest }) => { - const { passed, failed, outputLines, skipped } = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { projects: [ - { name: 'suite1' }, - { name: 'suite2' }, - { name: 'suite1' }, - { name: 'suite4' }, - ] }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('pass', async ({}, testInfo) => { - console.log('%%' + test.info().project.name); - }); - ` - }, { project: ['suite1', 'sUIte4'] }); - expect(passed).toBe(3); - expect(failed).toBe(0); - expect(skipped).toBe(0); - expect(new Set(outputLines)).toEqual(new Set(['suite1', 'suite1', 'suite4'])); -}); - test('should print nice error when some of the projects are unknown', async ({ runInlineTest }) => { const { output, exitCode } = await runInlineTest({ 'playwright.config.ts': ` diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index 2cba3b0623..6dfd74f442 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -388,7 +388,7 @@ test('should match snapshots from multiple projects', async ({ runInlineTest }) expect('Snapshot2').toMatchSnapshot('snapshot.txt'); }); `, - 'p2/a.spec.js-snapshots/snapshot.txt': `Snapshot2`, + 'p2/a.spec.js-snapshots/snapshot-project-1.txt': `Snapshot2`, }); expect(result.exitCode).toBe(0); }); diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 15c17743e4..3a264c3416 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -1207,7 +1207,7 @@ test('preserve reportName on projects', async ({ runInlineTest, mergeReports }) onBegin(config, suite) { const projects = suite.suites.map(s => s.project()).sort((a, b) => a.metadata.reportName.localeCompare(b.metadata.reportName)); console.log('projectNames: ' + projects.map(p => p.name)); - console.log('reportNames: ' + projects.map(p => p.metadata.reportName)); + suite.allTests().forEach(t => console.log('tags: ' + t.tags.join(' '))); } } module.exports = EchoReporter; @@ -1232,8 +1232,9 @@ test('preserve reportName on projects', async ({ runInlineTest, mergeReports }) const reportDir = test.info().outputPath('blob-report'); const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js')] }); expect(exitCode).toBe(0); - expect(output).toContain(`projectNames: foo,foo`); - expect(output).toContain(`reportNames: first,second`); + expect(output).toContain(`projectNames: foo`); + expect(output).toContain(`tags: @first @smoke`); + expect(output).toContain(`tags: @second @smoke`); }); test('no reports error', async ({ runInlineTest, mergeReports }) => { diff --git a/tests/playwright-test/test-output-dir.spec.ts b/tests/playwright-test/test-output-dir.spec.ts index 6d74551303..58852836e5 100644 --- a/tests/playwright-test/test-output-dir.spec.ts +++ b/tests/playwright-test/test-output-dir.spec.ts @@ -153,7 +153,6 @@ test('should include the project name', async ({ runInlineTest }) => { module.exports = { projects: [ {}, { name: 'foo' }, - { name: 'foo' }, { name: 'Bar space!' }, ] }; `, @@ -181,18 +180,12 @@ test('should include the project name', async ({ runInlineTest }) => { expect(result.output).toContain('test-results/my-test-test-1-retry1/bar.txt'); expect(result.output).toContain('my-test.spec.js-snapshots/bar.txt'); - // test1, run with foo #1 + // test1, run with foo expect(result.output).toContain('test-results/my-test-test-1-foo/bar.txt'); expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo.txt'); expect(result.output).toContain('test-results/my-test-test-1-foo-retry1/bar.txt'); expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo.txt'); - // test1, run with foo #2 - expect(result.output).toContain('test-results/my-test-test-1-foo1/bar.txt'); - expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo.txt'); - expect(result.output).toContain('test-results/my-test-test-1-foo1-retry1/bar.txt'); - expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo.txt'); - // test1, run with bar expect(result.output).toContain('test-results/my-test-test-1-Bar-space-/bar.txt'); expect(result.output).toContain('my-test.spec.js-snapshots/bar-Bar-space-.txt'); @@ -203,14 +196,10 @@ test('should include the project name', async ({ runInlineTest }) => { expect(result.output).toContain('test-results/my-test-test-2/bar.txt'); expect(result.output).toContain('my-test.spec.js-snapshots/bar-suffix.txt'); - // test2, run with foo #1 + // test2, run with foo expect(result.output).toContain('test-results/my-test-test-2-foo/bar.txt'); expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo-suffix.txt'); - // test2, run with foo #2 - expect(result.output).toContain('test-results/my-test-test-2-foo1/bar.txt'); - expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo-suffix.txt'); - // test2, run with bar expect(result.output).toContain('test-results/my-test-test-2-Bar-space-/bar.txt'); expect(result.output).toContain('my-test.spec.js-snapshots/bar-Bar-space--suffix.txt');