From 77deca1d6bbc6e6ca3173253313d749e81cffba0 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Thu, 8 Jul 2021 17:16:36 -0700 Subject: [PATCH] feat(test runner): export testInfo.data (#7525) This is a key-value storage for any information that goes into the report. Also export JSONReport types. --- src/test/dispatcher.ts | 1 + src/test/index.ts | 1 + src/test/ipc.ts | 1 + src/test/reporter.ts | 1 + src/test/reporters/json.ts | 72 +++++++++++++++---- src/test/test.ts | 1 + src/test/workerRunner.ts | 2 + tests/playwright-test/access-data.spec.ts | 18 +++++ .../playwright-test-fixtures.ts | 6 +- types/test.d.ts | 7 +- 10 files changed, 92 insertions(+), 18 deletions(-) diff --git a/src/test/dispatcher.ts b/src/test/dispatcher.ts index 2d0e76445a..ffee59e329 100644 --- a/src/test/dispatcher.ts +++ b/src/test/dispatcher.ts @@ -275,6 +275,7 @@ export class Dispatcher { const { test, result } = this._testById.get(params.testId)!; result.duration = params.duration; result.error = params.error; + result.data = params.data; test.expectedStatus = params.expectedStatus; test.annotations = params.annotations; test.timeout = params.timeout; diff --git a/src/test/index.ts b/src/test/index.ts index d348008c47..f7b55a4567 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -162,6 +162,7 @@ export const test = _baseTest.extend & { + projects: { + outputDir: string, + repeatEach: number, + retries: number, + metadata: any, + name: string, + testDir: string, + testIgnore: string[], + testMatch: string[], + timeout: number, + }[], + }; + suites: JSONReportSuite[]; + errors: TestError[]; +} +export interface JSONReportSuite { title: string; file: string; column: number; line: number; - specs: ReturnType[]; - suites?: SerializedSuite[]; + specs: JSONReportSpec[]; + suites?: JSONReportSuite[]; } - -export type ReportFormat = ReturnType; +export interface JSONReportSpec { + title: string; + ok: boolean; + tests: JSONReportTest[]; + file: string; + line: number; + column: number; +} +export interface JSONReportTest { + timeout: number; + annotations: { type: string, description?: string }[], + expectedStatus: TestStatus; + projectName: string; + results: JSONReportTestResult[]; + status: 'skipped' | 'expected' | 'unexpected' | 'flaky'; +} +export interface JSONReportTestResult { + workerIndex: number; + status: TestStatus | undefined; + duration: number; + error: TestError | undefined; + stdout: JSONReportSTDIOEntry[], + stderr: JSONReportSTDIOEntry[], + retry: number; + data: { [key: string]: any }, +} +export type JSONReportSTDIOEntry = { text: string } | { buffer: string }; function toPosixPath(aPath: string): string { return aPath.split(path.sep).join(path.posix.sep); @@ -58,7 +100,7 @@ class JSONReporter extends EmptyReporter { outputReport(this._serializeReport(), this._outputFile); } - private _serializeReport() { + private _serializeReport(): JSONReport { return { config: { ...this.config, @@ -77,15 +119,15 @@ class JSONReporter extends EmptyReporter { }; }) }, - suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s), + suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[], errors: this._errors }; } - private _serializeSuite(suite: Suite): null | SerializedSuite { + private _serializeSuite(suite: Suite): null | JSONReportSuite { if (!suite.findSpec(test => true)) return null; - const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as SerializedSuite[]; + const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as JSONReportSuite[]; return { title: suite.title, file: toPosixPath(path.relative(this.config.rootDir, suite.file)), @@ -96,7 +138,7 @@ class JSONReporter extends EmptyReporter { }; } - private _serializeTestSpec(spec: Spec) { + private _serializeTestSpec(spec: Spec): JSONReportSpec { return { title: spec.title, ok: spec.ok(), @@ -107,17 +149,18 @@ class JSONReporter extends EmptyReporter { }; } - private _serializeTest(test: Test) { + private _serializeTest(test: Test): JSONReportTest { return { timeout: test.timeout, annotations: test.annotations, expectedStatus: test.expectedStatus, projectName: test.projectName, results: test.results.map(r => this._serializeTestResult(r)), + status: test.status(), }; } - private _serializeTestResult(result: TestResult) { + private _serializeTestResult(result: TestResult): JSONReportTestResult { return { workerIndex: result.workerIndex, status: result.status, @@ -126,11 +169,12 @@ class JSONReporter extends EmptyReporter { stdout: result.stdout.map(s => stdioEntry(s)), stderr: result.stderr.map(s => stdioEntry(s)), retry: result.retry, + data: result.data, }; } } -function outputReport(report: ReportFormat, outputFile: string | undefined) { +function outputReport(report: JSONReport, outputFile: string | undefined) { const reportString = JSON.stringify(report, undefined, 2); outputFile = outputFile || process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]; if (outputFile) { diff --git a/src/test/test.ts b/src/test/test.ts index 60df087249..e3cf4e5a5c 100644 --- a/src/test/test.ts +++ b/src/test/test.ts @@ -227,6 +227,7 @@ export class Test implements reporterTypes.Test { duration: 0, stdout: [], stderr: [], + data: {}, }; this.results.push(result); return result; diff --git a/src/test/workerRunner.ts b/src/test/workerRunner.ts index 942358ce64..463f6c4120 100644 --- a/src/test/workerRunner.ts +++ b/src/test/workerRunner.ts @@ -225,6 +225,7 @@ export class WorkerRunner extends EventEmitter { retry: entry.retry, expectedStatus: 'passed', annotations: [], + data: {}, duration: 0, status: 'passed', stdout: [], @@ -484,6 +485,7 @@ function buildTestEndPayload(testId: string, testInfo: TestInfo): TestEndPayload expectedStatus: testInfo.expectedStatus, annotations: testInfo.annotations, timeout: testInfo.timeout, + data: testInfo.data, }; } diff --git a/tests/playwright-test/access-data.spec.ts b/tests/playwright-test/access-data.spec.ts index ff081d3fa2..f9b7b6572b 100644 --- a/tests/playwright-test/access-data.spec.ts +++ b/tests/playwright-test/access-data.spec.ts @@ -82,3 +82,21 @@ test('should report projectName in result', async ({ runInlineTest }) => { expect(report.suites[0].specs[0].tests[1].projectName).toBe(''); expect(exitCode).toBe(0); }); + +test('should access testInfo.data in fixture', async ({ runInlineTest }) => { + const { exitCode, report } = await runInlineTest({ + 'test-data-visible-in-env.spec.ts': ` + const test = pwt.test.extend({ + foo: async ({}, run, testInfo) => { + await run(); + testInfo.data.foo = 'bar'; + }, + }); + test('ensure fixture can set data', async ({ foo }) => { + }); + ` + }); + expect(exitCode).toBe(0); + const test = report.suites[0].specs[0].tests[0]; + expect(test.results[0].data).toEqual({ foo: 'bar' }); +}); diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index e84344f5b2..98a98be9ed 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -19,7 +19,7 @@ import { spawn } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; -import type { ReportFormat } from '../../src/test/reporters/json'; +import type { JSONReport, JSONReportSuite } from '../../src/test/reporters/json'; import rimraf from 'rimraf'; import { promisify } from 'util'; import * as url from 'url'; @@ -33,7 +33,7 @@ type RunResult = { failed: number, flaky: number, skipped: number, - report: ReportFormat, + report: JSONReport, results: any[], }; @@ -191,7 +191,7 @@ async function runPlaywrightTest(baseDir: string, params: any, env: Env, options } const results = []; - function visitSuites(suites?: ReportFormat['suites']) { + function visitSuites(suites?: JSONReportSuite[]) { if (!suites) return; for (const suite of suites) { diff --git a/types/test.d.ts b/types/test.d.ts index d739c83083..37c701c731 100644 --- a/types/test.d.ts +++ b/types/test.d.ts @@ -124,7 +124,7 @@ export type WebServerConfig = { */ command: string, /** - * The port that your server is expected to appear on. If not specified, it does get automatically collected via the + * The port that your server is expected to appear on. If not specified, it does get automatically collected via the * command output when a localhost URL gets printed. */ port?: number, @@ -394,6 +394,11 @@ export interface TestInfo extends WorkerInfo { */ annotations: { type: string, description?: string }[]; + /** + * Arbitrary data that test fixtures can provide for the test report. + */ + data: { [key: string]: any }; + /** * When tests are run multiple times, each run gets a unique `repeatEachIndex`. */