Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(test-runner): report nested steps (#8266)

Browse source
This commit is contained in:
Pavel Feldman 2021-08-17 13:57:26 -07:00 committed by GitHub
parent de4464cb9a
commit 97ba4f22f3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 173 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
docs/src/test-reporter-api/class-teststep.md
View file

@ -26,6 +26,11 @@ An error thrown during the step execution, if any.
Start time of this particular test step.
## property: TestStep.steps
- type: <[Array]<[TestStep]>>
List of steps inside this step.
## property: TestStep.title
- type: <[string]>

15
src/test/dispatcher.ts
View file

@ -35,7 +35,7 @@ export class Dispatcher {
private _freeWorkers: Worker[] = [];
private _workerClaimers: (() => void)[] = [];
private _testById = new Map<string, { test: TestCase, result: TestResult, steps: Map<string, TestStep> }>();
private _testById = new Map<string, { test: TestCase, result: TestResult, steps: Map<string, TestStep>, stepStack: Set<TestStep> }>();
private _queue: TestGroup[] = [];
private _stopCallback = () => {};
readonly _loader: Loader;
@ -52,7 +52,7 @@ export class Dispatcher {
for (const test of group.tests) {
const result = test._appendTestResult();
// When changing this line, change the one in retry too.
this._testById.set(test._id, { test, result, steps: new Map() });
this._testById.set(test._id, { test, result, steps: new Map(), stepStack: new Set() });
}
}
}
@ -197,6 +197,7 @@ export class Dispatcher {
if (!this._isStopped && pair.test.expectedStatus === 'passed' && pair.test.results.length < pair.test.retries + 1) {
pair.result = pair.test._appendTestResult();
pair.steps = new Map();
pair.stepStack = new Set();
remaining.push(pair.test);
}
}
@ -280,19 +281,22 @@ export class Dispatcher {
this._reportTestEnd(test, result);
});
worker.on('stepBegin', (params: StepBeginPayload) => {
const { test, result, steps } = this._testById.get(params.testId)!;
const { test, result, steps, stepStack } = this._testById.get(params.testId)!;
const step: TestStep = {
title: params.title,
category: params.category,
startTime: new Date(params.wallTime),
duration: 0,
steps: [],
};
steps.set(params.stepId, step);
result.steps.push(step);
const parentStep = [...stepStack].pop() || result;
parentStep.steps.push(step);
stepStack.add(step);
this._reporter.onStepBegin?.(test, result, step);
});
worker.on('stepEnd', (params: StepEndPayload) => {
const { test, result, steps } = this._testById.get(params.testId)!;
const { test, result, steps, stepStack } = this._testById.get(params.testId)!;
const step = steps.get(params.stepId);
if (!step) {
this._reporter.onStdErr?.('Internal error: step end without step begin: ' + params.stepId, test, result);
@ -301,6 +305,7 @@ export class Dispatcher {
step.duration = params.wallTime - step.startTime.getTime();
if (params.error)
step.error = params.error;
stepStack.delete(step);
steps.delete(params.stepId);
this._reporter.onStepEnd?.(test, result, step);
});

48
src/test/reporters/html.ts
View file

@ -199,7 +199,7 @@ class HtmlReporter {
attachments: await this._createAttachments(testId, result),
stdout: result.stdout,
stderr: result.stderr,
steps: this._serializeSteps(result.steps)
steps: serializeSteps(result.steps)
};
}
@ -249,43 +249,19 @@ class HtmlReporter {
sha1
};
}
private _serializeSteps(steps: TestStep[]): JsonTestStep[] {
const stepStack: TestStep[] = [];
const result: JsonTestStep[] = [];
const stepMap = new Map<TestStep, JsonTestStep>();
for (const step of steps) {
let lastStep = stepStack[stepStack.length - 1];
while (lastStep && !containsStep(lastStep, step)) {
stepStack.pop();
lastStep = stepStack[stepStack.length - 1];
}
const collection = stepMap.get(lastStep!)?.steps || result;
const jsonStep = {
title: step.title,
category: step.category,
startTime: step.startTime.toISOString(),
duration: step.duration,
error: step.error,
steps: []
};
collection.push(jsonStep);
stepMap.set(step, jsonStep);
stepStack.push(step);
}
return result;
}
}
function containsStep(outer: TestStep, inner: TestStep): boolean {
if (outer.startTime.getTime() > inner.startTime.getTime())
return false;
if (outer.startTime.getTime() + outer.duration < inner.startTime.getTime() + inner.duration)
return false;
if (outer.startTime.getTime() + outer.duration <= inner.startTime.getTime())
return false;
return true;
function serializeSteps(steps: TestStep[]): JsonTestStep[] {
return steps.map(step => {
return {
title: step.title,
category: step.category,
startTime: step.startTime.toISOString(),
duration: step.duration,
error: step.error,
steps: serializeSteps(step.steps),
};
});
}
function isTextAttachment(contentType: string) {

149
tests/playwright-test/reporter.spec.ts
View file

@ -165,11 +165,11 @@ test('should report expect steps', async ({ runInlineTest }) => {
process.stdout.write(chunk);
}
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
console.log('%%%% begin', JSON.stringify(copy));
}
onStepEnd(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
if (copy.error?.stack)
copy.error.stack = '<stack>';
console.log('%%%% end', JSON.stringify(copy));
@ -244,11 +244,11 @@ test('should report api steps', async ({ runInlineTest }) => {
console.log('%%%% test end ' + test.title);
}
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
console.log('%%%% begin', JSON.stringify(copy));
}
onStepEnd(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
if (copy.error?.stack)
copy.error.stack = '<stack>';
console.log('%%%% end', JSON.stringify(copy));
@ -335,11 +335,11 @@ test('should report api step failure', async ({ runInlineTest }) => {
process.stdout.write(chunk);
}
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
console.log('%%%% begin', JSON.stringify(copy));
}
onStepEnd(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
if (copy.error?.stack)
copy.error.stack = '<stack>';
console.log('%%%% end', JSON.stringify(copy));
@ -388,11 +388,11 @@ test('should report test.step', async ({ runInlineTest }) => {
process.stdout.write(chunk);
}
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
console.log('%%%% begin', JSON.stringify(copy));
}
onStepEnd(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
const copy = { ...step, startTime: undefined, duration: undefined, steps: undefined };
if (copy.error?.stack)
copy.error.stack = '<stack>';
console.log('%%%% end', JSON.stringify(copy));
@ -435,6 +435,139 @@ test('should report test.step', async ({ runInlineTest }) => {
]);
});
test('should report api step hierarchy', async ({ runInlineTest }) => {
const expectReporterJS = `
class Reporter {
onBegin(config: FullConfig, suite: Suite) {
this.suite = suite;
}
async onEnd() {
const processSuite = (suite: Suite) => {
for (const child of suite.suites)
processSuite(child);
for (const test of suite.tests) {
for (const result of test.results) {
for (const step of result.steps) {
console.log('%% ' + JSON.stringify(step));
}
}
}
};
processSuite(this.suite);
}
}
module.exports = Reporter;
`;
const result = await runInlineTest({
'reporter.ts': expectReporterJS,
'playwright.config.ts': `
module.exports = {
reporter: './reporter',
};
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async ({ page }) => {
await test.step('outer step 1', async () => {
await test.step('inner step 1.1', async () => {});
await test.step('inner step 1.2', async () => {});
});
await test.step('outer step 2', async () => {
await test.step('inner step 2.1', async () => {});
await test.step('inner step 2.2', async () => {});
});
});
`
}, { reporter: '', workers: 1 });
expect(result.exitCode).toBe(0);
const objects = result.output.split('\n').filter(line => line.startsWith('%% ')).map(line => line.substring(3).trim()).filter(Boolean).map(line => JSON.parse(line));
const distill = step => {
step.duration = 1;
step.startTime = 'time';
step.steps.forEach(distill);
};
objects.forEach(distill);
expect(objects).toEqual([
{
category: 'hook',
title: 'Before Hooks',
duration: 1,
startTime: 'time',
steps: [
{
category: 'pw:api',
title: 'browserContext.newPage',
duration: 1,
startTime: 'time',
steps: [],
},
],
},
{
category: 'test.step',
title: 'outer step 1',
duration: 1,
startTime: 'time',
steps: [
{
category: 'test.step',
title: 'inner step 1.1',
duration: 1,
startTime: 'time',
steps: [],
},
{
category: 'test.step',
title: 'inner step 1.2',
duration: 1,
startTime: 'time',
steps: [],
},
],
},
{
category: 'test.step',
title: 'outer step 2',
duration: 1,
startTime: 'time',
steps: [
{
category: 'test.step',
title: 'inner step 2.1',
duration: 1,
startTime: 'time',
steps: [],
},
{
category: 'test.step',
title: 'inner step 2.2',
duration: 1,
startTime: 'time',
steps: [],
},
],
},
{
category: 'hook',
title: 'After Hooks',
duration: 1,
startTime: 'time',
steps: [
{
category: 'pw:api',
title: 'browserContext.close',
duration: 1,
startTime: 'time',
steps: [],
},
],
},
]);
});
function stripEscapedAscii(str: string) {
return str.replace(/\\u00[a-z0-9][a-z0-9]\[[^m]+m/g, '');
}

4
types/testReporter.d.ts vendored
View file

@ -246,6 +246,10 @@ export interface TestStep {
* An error thrown during the step execution, if any.
*/
error?: TestError;
/**
* List of steps inside this step.
*/
steps: TestStep[];
}
/**

1
utils/generate_types/overrides-testReporter.d.ts vendored
View file

@ -64,6 +64,7 @@ export interface TestStep {
startTime: Date;
duration: number;
error?: TestError;
steps: TestStep[];
}
/**