fix(test runner): account for errors with inconsistent stack/message (#8950)

src/test/reporters/base.ts

@@ -243,22 +243,20 @@ export function formatError(error: TestError, file?: string) {
   const tokens = [];
   if (stack) {
     tokens.push('');
-    const message = error.message || '';
+    const lines = stack.split('\n');
-    const messageLocation = stack.indexOf(message);
+    let firstStackLine = lines.findIndex(line => line.startsWith('    at '));
-    const preamble = stack.substring(0, messageLocation + message.length);
+    if (firstStackLine === -1)
-    tokens.push(preamble);
+      firstStackLine = lines.length;
-    const position = file ? positionInFile(stack, file) : null;
+    tokens.push(lines.slice(0, firstStackLine).join('\n'));
+    const stackLines = lines.slice(firstStackLine);
+    const position = file ? positionInFile(stackLines, file) : null;
     if (position) {
       const source = fs.readFileSync(file!, 'utf8');
       tokens.push('');
-      tokens.push(codeFrameColumns(source, {
+      tokens.push(codeFrameColumns(source, { start: position }, { highlightCode: colors.enabled }));
-        start: position,
-      },
-      { highlightCode: colors.enabled }
-      ));
     }
     tokens.push('');
-    tokens.push(colors.dim(preamble.length > 0 ? stack.substring(preamble.length + 1) : stack));
+    tokens.push(colors.dim(stackLines.join('\n')));
   } else if (error.message) {
     tokens.push('');
     tokens.push(error.message);
@@ -279,10 +277,10 @@ function indent(lines: string, tab: string) {
   return lines.replace(/^(?=.+$)/gm, tab);
 }
 
-function positionInFile(stack: string, file: string): { column: number; line: number; } | undefined {
+function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined {
   // Stack will have /private/var/folders instead of /var/folders on Mac.
   file = fs.realpathSync(file);
-  for (const line of stack.split('\n')) {
+  for (const line of stackLines) {
     const parsed = stackUtils.parseLine(line);
     if (!parsed || !parsed.file)
       continue;

tests/playwright-test/base-reporter.spec.ts

@@ -226,3 +226,24 @@ test('should print stack-less errors', async ({ runInlineTest }) => {
   expect(result.failed).toBe(1);
   expect(result.output).toContain('Hello');
 });
+
+test('should print errors with inconsistent message/stack', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('foobar', async function myTest({}) {
+        const e = new Error('Hello');
+        // Force stack to contain "Hello".
+        // Otherwise it is computed lazy and will get 'foo bar' instead.
+        e.stack;
+        e.message = 'foo bar';
+        e.stack = 'hi!' + e.stack;
+        throw e;
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('hi!Error: Hello');
+  expect(result.output).toContain('at myTest');
+});

tests/playwright-test/playwright.expect.text.spec.ts

@@ -231,3 +231,23 @@ test('should print expected/received before timeout', async ({ runInlineTest })
   expect(stripAscii(result.output)).toContain('Expected string: "Text 2"');
   expect(stripAscii(result.output)).toContain('Received string: "Text content"');
 });
+
+test('should print nice error for toHaveText', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = pwt;
+
+      test('fail', async ({ page }) => {
+        await page.setContent('<div id=node>Text content</div>');
+        await expect(page.locator('no-such-thing')).toHaveText('Text');
+      });
+      `,
+  }, { workers: 1, timeout: 2000 });
+  expect(result.failed).toBe(1);
+  expect(result.exitCode).toBe(1);
+  const output = stripAscii(result.output);
+  expect(output).toContain('Pending operations:');
+  expect(output).toContain('Error: expect(received).toHaveText(expected)');
+  expect(output).toContain('Expected string: "Text"');
+  expect(output).toContain('Received string: undefined');
+});