feat(test-runner): support expect.soft (#11800)

Soft expects will still fail the test, but will not abort it's execution. As a consequence of this:
-  `TestResult` now might have multiple errors, which is reflected with a new `testResult.erros: TestError[]` field.
- `TestInfo` now might have multiple errors as well, which is reflected with a new `testInfo.errors: TestError[]` field.

Fixes #7819

docs/src/test-api/class-testinfo.md

@@ -128,12 +128,15 @@ The number of milliseconds the test took to finish. Always zero before the test
 
 
 ## property: TestInfo.error
-- type: <[Object]>
-  - `message` <[void]|[string]> Error message. Set when `Error` (or its subclass) has been thrown.
-  - `stack` <[void]|[string]> Error stack. Set when `Error` (or its subclass) has been thrown.
-  - `value` <[void]|[string]> The thrown value. Set when anything except the `Error` (or its subclass) has been thrown.
+- type: <[void]|[TestError]>
 
-An error thrown during test execution, if any.
+First error thrown during test execution, if any. This is equal to the first
+element in [`property: TestInfo.errors`].
+
+## property: TestInfo.errors
+- type: <[Array]<[TestError]>>
+
+Errors thrown during test execution, if any.
 
 
 ## property: TestInfo.expectedStatus

docs/src/test-assertions-js.md

@@ -19,10 +19,25 @@ expect(value).not.toEqual(0);
 await expect(locator).not.toContainText("some text");
 ```
 
-You can also specify a custom error message as a second argument to the `expect` function, for example:
+By default, failed assertion will terminate test execution. Playwright also
+supports *soft assertions*: failed soft assertions **do not** terminate test execution,
+but mark the test as failed.
+
+```js
+// Make a few checks that will not stop the test when failed...
+await expect.soft(page.locator('#status')).toHaveText('Success');
+await expect.soft(page.locator('#eta')).toHaveText('1 day');
+
+// ... and continue the test to check more things.
+await page.locator('#next-page').click();
+await expect.soft(page.locator('#title')).toHaveText('Make another order');
+```
+
+You can specify a custom error message as a second argument to the `expect` function, for example:
 
 ```js
 expect(value, 'my custom error message').toBe(42);
+expect.soft(value, 'my soft assertion').toBe(56);
 ```
 
 <!-- TOC -->

docs/src/test-reporter-api/class-testresult.md

@@ -20,7 +20,13 @@ Running time in milliseconds.
 ## property: TestResult.error
 - type: <[void]|[TestError]>
 
-An error thrown during the test execution, if any.
+First error thrown during test execution, if any. This is equal to the first
+element in [`property: TestResult.errors`].
+
+## property: TestResult.errors
+- type: <[Array]<[TestError]>>
+
+Errors thrown during the test execution.
 
 ## property: TestResult.retry
 - type: <[int]>

packages/html-reporter/src/testCaseView.spec.tsx

@@ -26,6 +26,7 @@ const result: TestResult = {
   retry: 0,
   startTime: new Date(0).toUTCString(),
   duration: 100,
+  errors: [],
   steps: [{
     title: 'Outer step',
     startTime: new Date(100).toUTCString(),

packages/html-reporter/src/testResultView.css

@@ -55,6 +55,7 @@
   border-radius: 6px;
   padding: 16px;
   line-height: initial;
+  margin-bottom: 6px;
 }
 
 .test-result-counter {

packages/html-reporter/src/testResultView.tsx

@@ -51,8 +51,8 @@ export const TestResultView: React.FC<{
   const diff = attachmentsMap.get('diff');
   const hasImages = [actual?.contentType, expected?.contentType, diff?.contentType].some(v => v && /^image\//i.test(v));
   return <div className='test-result'>
-    {result.error && <AutoChip header='Errors'>
-      <ErrorMessage key='test-result-error-message' error={result.error}></ErrorMessage>
+    {!!result.errors.length && <AutoChip header='Errors'>
+      {result.errors.map((error, index) => <ErrorMessage key={'test-result-error-message-' + index} error={error}></ErrorMessage>)}
     </AutoChip>}
     {!!result.steps.length && <AutoChip header='Test Steps'>
       {result.steps.map((step, i) => <StepTreeItem key={`step-${i}`} step={step} depth={0}></StepTreeItem>)}

packages/playwright-test/src/dispatcher.ts

@@ -199,7 +199,8 @@ export class Dispatcher {
       const { result } = data.resultByWorkerIndex.get(worker.workerIndex)!;
       data.resultByWorkerIndex.delete(worker.workerIndex);
       result.duration = params.duration;
-      result.error = params.error;
+      result.errors = params.errors;
+      result.error = result.errors[0];
       result.attachments = params.attachments.map(a => ({
         name: a.name,
         path: a.path,
@@ -292,7 +293,8 @@ export class Dispatcher {
         if (runningHookId) {
           const data = this._testById.get(runningHookId)!;
           const { result } = data.resultByWorkerIndex.get(worker.workerIndex)!;
-          result.error = params.fatalError;
+          result.errors = [params.fatalError];
+          result.error = result.errors[0];
           result.status = 'failed';
           this._reporter.onTestEnd?.(data.test, result);
         }
@@ -312,7 +314,8 @@ export class Dispatcher {
             if (test._type === 'test')
               this._reporter.onTestBegin?.(test, result);
           }
-          result.error = params.fatalError;
+          result.errors = [params.fatalError];
+          result.error = result.errors[0];
           result.status = first ? 'failed' : 'skipped';
           this._reportTestEnd(test, result);
           failedTestIds.add(test._id);

packages/playwright-test/src/expect.ts

@@ -92,15 +92,23 @@ export const printReceivedStringContainExpectedResult = (
 
 // #endregion
 
+function createExpect(actual: unknown, message: string|undefined, isSoft: boolean) {
+  if (message !== undefined && typeof message !== 'string')
+    throw new Error('expect(actual, optionalErrorMessage): optional error message must be a string.');
+  return new Proxy(expectLibrary(actual), new ExpectMetaInfoProxyHandler(message || '', isSoft));
+}
+
 export const expect: Expect = new Proxy(expectLibrary as any, {
   apply: function(target: any, thisArg: any, argumentsList: [actual: unknown, message: string|undefined]) {
-    const message = argumentsList[1];
-    if (message !== undefined && typeof message !== 'string')
-      throw new Error('expect(actual, optionalErrorMessage): optional error message must be a string.');
-    return new Proxy(expectLibrary.call(thisArg, argumentsList[0]), new ExpectMetaInfoProxyHandler(message || ''));
+    const [actual, message] = argumentsList;
+    return createExpect(actual, message, false /* isSoft */);
   }
 });
 
+expect.soft = (actual: unknown, message: string|undefined) => {
+  return createExpect(actual, message, true /* isSoft */);
+};
+
 expectLibrary.setState({ expand: false });
 const customMatchers = {
   toBeChecked,
@@ -128,15 +136,18 @@ const customMatchers = {
 
 type ExpectMetaInfo = {
   message: string;
+  isSoft: boolean;
 };
 
 let expectCallMetaInfo: undefined|ExpectMetaInfo = undefined;
 
 class ExpectMetaInfoProxyHandler {
   private _message: string;
+  private _isSoft: boolean;
 
-  constructor(message: string) {
+  constructor(message: string, isSoft: boolean) {
     this._message = message;
+    this._isSoft = isSoft;
   }
 
   get(target: any, prop: any, receiver: any): any {
@@ -144,12 +155,26 @@ class ExpectMetaInfoProxyHandler {
     if (typeof value !== 'function')
       return new Proxy(value, this);
     return (...args: any[]) => {
+      const testInfo = currentTestInfo();
+      if (!testInfo)
+        return value.call(target, ...args);
+      const handleError = (e: Error) => {
+        if (this._isSoft)
+          testInfo._failWithError(serializeError(e), false /* isHardError */);
+        else
+          throw e;
+      };
       try {
         expectCallMetaInfo = {
           message: this._message,
+          isSoft: this._isSoft,
         };
-        const result = value.call(target, ...args);
+        let result = value.call(target, ...args);
+        if ((result instanceof Promise))
+          result = result.catch(handleError);
         return result;
+      } catch (e) {
+        handleError(e);
       } finally {
         expectCallMetaInfo = undefined;
       }
@@ -172,10 +197,11 @@ function wrap(matcherName: string, matcher: any) {
     const stackLines = new Error().stack!.split('\n').slice(INTERNAL_STACK_LENGTH + 1);
     const frame = stackLines[0] ? stackUtils.parseLine(stackLines[0]) : undefined;
     const customMessage = expectCallMetaInfo?.message ?? '';
+    const isSoft = expectCallMetaInfo?.isSoft ?? false;
     const step = testInfo._addStep({
       location: frame && frame.file ? { file: path.resolve(process.cwd(), frame.file), line: frame.line || 0, column: frame.column || 0 } : undefined,
       category: 'expect',
-      title: customMessage || `expect${this.isNot ? '.not' : ''}.${matcherName}`,
+      title: customMessage || `expect${isSoft ? '.soft' : ''}${this.isNot ? '.not' : ''}.${matcherName}`,
       canHaveChildren: true,
       forceNoParent: false
     });

packages/playwright-test/src/index.ts

@@ -380,7 +380,8 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
       const anyContext = leftoverContexts[0];
       const pendingCalls = anyContext ? formatPendingCalls((anyContext as any)._connection.pendingProtocolCalls()) : '';
       await Promise.all(leftoverContexts.filter(c => createdContexts.has(c)).map(c => c.close()));
-      testInfo.error = prependToTestError(testInfo.error, pendingCalls);
+      if (pendingCalls)
+        testInfo.error = prependToTestError(testInfo.error, pendingCalls);
     }
   }, { auto: true }],
 
@@ -434,7 +435,8 @@ export const test = _baseTest.extend<TestFixtures, WorkerFixtures>({
       }
     }));
 
-    testInfo.error = prependToTestError(testInfo.error, prependToError);
+    if (prependToError)
+      testInfo.error = prependToTestError(testInfo.error, prependToError);
   },
 
   context: async ({ _contextFactory }, use) => {

packages/playwright-test/src/ipc.ts

@@ -39,7 +39,7 @@ export type TestEndPayload = {
   testId: string;
   duration: number;
   status: TestStatus;
-  error?: TestError;
+  errors: TestError[];
   expectedStatus: TestStatus;
   annotations: { type: string, description?: string }[];
   timeout: number;

packages/playwright-test/src/matchers/golden.ts

@@ -23,7 +23,7 @@ import jpeg from 'jpeg-js';
 import pixelmatch from 'pixelmatch';
 import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch';
 import { UpdateSnapshots } from '../types';
-import { addSuffixToFilePath } from '../util';
+import { addSuffixToFilePath, serializeError } from '../util';
 import BlinkDiff from '../third_party/blink-diff';
 import PNGImage from '../third_party/png-js';
 import { TestInfoImpl } from '../testInfo';
@@ -129,8 +129,8 @@ export function compare(
       return { pass: true, message };
     }
     if (updateSnapshots === 'missing') {
-      testInfo._appendErrorMessage(message);
-      return { pass: true, message };
+      testInfo._failWithError(serializeError(new Error(message)), false /* isHardError */);
+      return { pass: true };
     }
     return { pass: false, message };
   }

packages/playwright-test/src/reporters/base.ts

@@ -33,11 +33,6 @@ type Annotation = {
   location?: Location;
 };
 
-type FailureDetails = {
-  tokens: string[];
-  location?: Location;
-};
-
 type ErrorDetails = {
   message: string;
   location?: Location;
@@ -99,7 +94,7 @@ export class BaseReporter implements Reporter  {
   }
 
   onError(error: TestError) {
-    console.log(formatError(this.config, error, colors.enabled).message);
+    console.log('\n' + formatError(this.config, error, colors.enabled).message);
   }
 
   async onEnd(result: FullResult) {
@@ -232,14 +227,16 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
   lines.push(colors.red(header));
   for (const result of test.results) {
     const resultLines: string[] = [];
-    const { tokens: resultTokens, location } = formatResultFailure(config, test, result, '    ', colors.enabled);
-    if (!resultTokens.length)
+    const errors = formatResultFailure(config, test, result, '    ', colors.enabled);
+    if (!errors.length)
       continue;
+    const retryLines = [];
     if (result.retry) {
-      resultLines.push('');
-      resultLines.push(colors.gray(pad(`    Retry #${result.retry}`, '-')));
+      retryLines.push('');
+      retryLines.push(colors.gray(pad(`    Retry #${result.retry}`, '-')));
     }
-    resultLines.push(...resultTokens);
+    resultLines.push(...retryLines);
+    resultLines.push(...errors.map(error => '\n' + error.message));
     if (includeAttachments) {
       for (let i = 0; i < result.attachments.length; ++i) {
         const attachment = result.attachments[i];
@@ -277,11 +274,13 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
       resultLines.push('');
       resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-'));
     }
-    annotations.push({
-      location,
-      title,
-      message: [header, ...resultLines].join('\n'),
-    });
+    for (const error of errors) {
+      annotations.push({
+        location: error.location,
+        title,
+        message: [header, ...retryLines, error.message].join('\n'),
+      });
+    }
     lines.push(...resultLines);
   }
   lines.push('');
@@ -291,25 +290,27 @@ export function formatFailure(config: FullConfig, test: TestCase, options: {inde
   };
 }
 
-export function formatResultFailure(config: FullConfig, test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): FailureDetails {
-  const resultTokens: string[] = [];
+export function formatResultFailure(config: FullConfig, test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): ErrorDetails[] {
+  const errorDetails: ErrorDetails[] = [];
+
   if (result.status === 'timedOut') {
-    resultTokens.push('');
-    resultTokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), initialIndent));
+    errorDetails.push({
+      message: indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), initialIndent),
+    });
+  } else if (result.status === 'passed' && test.expectedStatus === 'failed') {
+    errorDetails.push({
+      message: indent(colors.red(`Expected to fail, but passed.`), initialIndent),
+    });
   }
-  if (result.status === 'passed' && test.expectedStatus === 'failed') {
-    resultTokens.push('');
-    resultTokens.push(indent(colors.red(`Expected to fail, but passed.`), initialIndent));
+
+  for (const error of result.errors) {
+    const formattedError = formatError(config, error, highlightCode, test.location.file);
+    errorDetails.push({
+      message: indent(formattedError.message, initialIndent),
+      location: formattedError.location,
+    });
   }
-  let error: ErrorDetails | undefined = undefined;
-  if (result.error !== undefined) {
-    error = formatError(config, result.error, highlightCode, test.location.file);
-    resultTokens.push(indent(error.message, initialIndent));
-  }
-  return {
-    tokens: resultTokens,
-    location: error?.location,
-  };
+  return errorDetails;
 }
 
 function relativeFilePath(config: FullConfig, file: string): string {
@@ -341,7 +342,7 @@ function formatTestHeader(config: FullConfig, test: TestCase, indent: string, in
 
 export function formatError(config: FullConfig, error: TestError, highlightCode: boolean, file?: string): ErrorDetails {
   const stack = error.stack;
-  const tokens = [''];
+  const tokens = [];
   let location: Location | undefined;
   if (stack) {
     const parsed = prepareErrorStack(stack, file);

packages/playwright-test/src/reporters/html.ts

@@ -93,7 +93,7 @@ export type TestResult = {
   startTime: string;
   duration: number;
   steps: TestStep[];
-  error?: string;
+  errors: string[];
   attachments: TestAttachment[];
   status: 'passed' | 'failed' | 'timedOut' | 'skipped';
 };
@@ -393,7 +393,7 @@ class HtmlBuilder {
       startTime: result.startTime,
       retry: result.retry,
       steps: result.steps.map(s => this._createTestStep(s)),
-      error: result.error,
+      errors: result.errors,
       status: result.status,
       attachments: result.attachments.map(a => {
         if (a.name === 'trace')

packages/playwright-test/src/reporters/raw.ts

@@ -83,7 +83,7 @@ export type JsonTestResult = {
   startTime: string;
   duration: number;
   status: TestStatus;
-  error?: JsonError;
+  errors: JsonError[];
   attachments: JsonAttachment[];
   steps: JsonTestStep[];
 };
@@ -224,7 +224,7 @@ class RawReporter {
       startTime: result.startTime.toISOString(),
       duration: result.duration,
       status: result.status,
-      error: formatResultFailure(this.config, test, result, '', true).tokens.join('').trim(),
+      errors: formatResultFailure(this.config, test, result, '', true).map(error => error.message),
       attachments: this._createAttachments(result),
       steps: dedupeSteps(result.steps.map(step => this._serializeStep(test, step)))
     };

packages/playwright-test/src/test.ts

@@ -191,7 +191,8 @@ export class TestCase extends Base implements reporterTypes.TestCase {
       stderr: [],
       attachments: [],
       status: 'skipped',
-      steps: []
+      steps: [],
+      errors: [],
     };
     this.results.push(result);
     return result;

packages/playwright-test/src/testInfo.ts

@@ -34,6 +34,7 @@ export class TestInfoImpl implements TestInfo {
   readonly _timeoutRunner: TimeoutRunner;
   readonly _startTime: number;
   readonly _startWallTime: number;
+  private _hasHardError: boolean = false;
 
   // ------------ TestInfo fields ------------
   readonly repeatEachIndex: number;
@@ -59,7 +60,20 @@ export class TestInfoImpl implements TestInfo {
   snapshotSuffix: string = '';
   readonly outputDir: string;
   readonly snapshotDir: string;
-  error: TestError | undefined = undefined;
+  errors: TestError[] = [];
+
+  get error(): TestError | undefined {
+    return this.errors.length > 0 ? this.errors[0] : undefined;
+  }
+
+  set error(e: TestError | undefined) {
+    if (e === undefined)
+      throw new Error('Cannot assign testInfo.error undefined value!');
+    if (!this.errors.length)
+      this.errors.push(e);
+    else
+      this.errors[0] = e;
+  }
 
   constructor(
     loader: Loader,
@@ -168,7 +182,7 @@ export class TestInfoImpl implements TestInfo {
           this.status = 'skipped';
       } else {
         const serialized = serializeError(error);
-        this._failWithError(serialized);
+        this._failWithError(serialized, true /* isHardError */);
         return serialized;
       }
     }
@@ -178,25 +192,18 @@ export class TestInfoImpl implements TestInfo {
     return this._addStepImpl(data);
   }
 
-  _failWithError(error: TestError) {
-    // Do not overwrite any previous error and error status.
+  _failWithError(error: TestError, isHardError: boolean) {
+    // Do not overwrite any previous hard errors.
     // Some (but not all) scenarios include:
     //   - expect() that fails after uncaught exception.
     //   - fail after the timeout, e.g. due to fixture teardown.
+    if (isHardError && this._hasHardError)
+      return;
+    if (isHardError)
+      this._hasHardError = true;
     if (this.status === 'passed')
       this.status = 'failed';
-    if (this.error === undefined)
-      this.error = error;
-  }
-
-  _appendErrorMessage(message: string) {
-    // Do not overwrite any previous error status.
-    if (this.status === 'passed')
-      this.status = 'failed';
-    if (this.error === undefined)
-      this.error = { value: 'Error: ' + message };
-    else if (this.error.value)
-      this.error.value += '\nError: ' + message;
+    this.errors.push(error);
   }
 
   // ------------ TestInfo methods ------------

packages/playwright-test/src/util.ts

@@ -24,6 +24,7 @@ import { calculateSha1, isRegExp } from 'playwright-core/lib/utils/utils';
 import { isInternalFileName } from 'playwright-core/lib/utils/stackTrace';
 
 const PLAYWRIGHT_CORE_PATH = path.dirname(require.resolve('playwright-core'));
+const EXPECT_PATH = path.dirname(require.resolve('expect'));
 const PLAYWRIGHT_TEST_PATH = path.join(__dirname, '..');
 
 function filterStackTrace(e: Error) {
@@ -46,7 +47,7 @@ function filterStackTrace(e: Error) {
       const functionName = callSite.getFunctionName() || undefined;
       if (!fileName)
         return true;
-      return !fileName.startsWith(PLAYWRIGHT_TEST_PATH) && !fileName.startsWith(PLAYWRIGHT_CORE_PATH) && !isInternalFileName(fileName, functionName);
+      return !fileName.startsWith(PLAYWRIGHT_TEST_PATH) && !fileName.startsWith(PLAYWRIGHT_CORE_PATH) && !fileName.startsWith(EXPECT_PATH) && !isInternalFileName(fileName, functionName);
     }));
   };
   // eslint-disable-next-line
@@ -202,9 +203,7 @@ export function getContainedPath(parentPath: string, subPath: string = ''): stri
 
 export const debugTest = debug('pw:test');
 
-export function prependToTestError(testError: TestError | undefined, message: string | undefined, location?: Location) {
-  if (!message)
-    return testError;
+export function prependToTestError(testError: TestError | undefined, message: string, location?: Location): TestError {
   if (!testError) {
     if (!location)
       return { value: message };

packages/playwright-test/src/workerRunner.ts

@@ -95,7 +95,7 @@ export class WorkerRunner extends EventEmitter {
     // and continuing to run tests in the same worker is problematic. Therefore,
     // we turn this into a fatal error and restart the worker anyway.
     if (this._currentTest && this._currentTest._test._type === 'test' && this._currentTest.expectedStatus !== 'failed') {
-      this._currentTest._failWithError(serializeError(error));
+      this._currentTest._failWithError(serializeError(error), true /* isHardError */);
     } else {
       // No current test - fatal error.
       if (!this._fatalError)
@@ -395,7 +395,7 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {
     testId: testInfo._test._id,
     duration: testInfo.duration,
     status: testInfo.status!,
-    error: testInfo.error,
+    errors: testInfo.errors,
     expectedStatus: testInfo.expectedStatus,
     annotations: testInfo.annotations,
     timeout: testInfo.timeout,

packages/playwright-test/types/test.d.ts

@@ -1492,9 +1492,14 @@ export interface TestInfo {
    */
   status?: TestStatus;
   /**
-   * An error thrown during test execution, if any.
+   * First error thrown during test execution, if any. This is equal to the first element in
+   * [testInfo.errors](https://playwright.dev/docs/api/class-testinfo#test-info-errors).
    */
   error?: TestError;
+  /**
+   * Errors thrown during test execution, if any.
+   */
+  errors: TestError[];
   /**
    * Output written to `process.stdout` or `console.log` during the test execution.
    */

packages/playwright-test/types/testExpect.d.ts

@@ -29,6 +29,7 @@ type MakeMatchers<T, ReturnValue = T> = PlaywrightTest.Matchers<ReturnValue> &
 
 export declare type Expect = {
   <T = unknown>(actual: T, message?: string): MakeMatchers<T>;
+  soft: <T = unknown>(actual: T, message?: string) => MakeMatchers<T>;
 
   // Sourced from node_modules/expect/build/types.d.ts
   assertions(arg0: number): void;

packages/playwright-test/types/testReporter.d.ts

@@ -213,9 +213,14 @@ export interface TestResult {
    */
   status: TestStatus;
   /**
-   * An error thrown during the test execution, if any.
+   * First error thrown during test execution, if any. This is equal to the first element in
+   * [testResult.errors](https://playwright.dev/docs/api/class-testresult#test-result-errors).
    */
   error?: TestError;
+  /**
+   * Errors thrown during the test execution.
+   */
+  errors: TestError[];
   /**
    * The list of files or buffers attached during the test execution through
    * [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments).

tests/playwright-test/expect-soft.spec.ts

@@ -0,0 +1,80 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAnsi } from './playwright-test-fixtures';
+
+test('soft expects should compile', async ({ runTSC }) => {
+  const result = await runTSC({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should work', () => {
+        test.expect.soft(1+1).toBe(3);
+        test.expect.soft(1+1, 'custom error message').toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('soft expects should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should work', () => {
+        test.expect.soft(1+1).toBe(3);
+        console.log('woof-woof');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('woof-woof');
+});
+
+test('should report a mixture of soft and non-soft errors', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should work', ({}) => {
+        test.expect.soft(1+1, 'one plus one').toBe(3);
+        test.expect.soft(2*2, 'two times two').toBe(5);
+        test.expect(3/3, 'three div three').toBe(7);
+        test.expect.soft(6-4, 'six minus four').toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('Error: one plus one');
+  expect(stripAnsi(result.output)).toContain('Error: two times two');
+  expect(stripAnsi(result.output)).toContain('Error: three div three');
+  expect(stripAnsi(result.output)).not.toContain('Error: six minus four');
+});
+
+test('testInfo should contain all soft expect errors', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should work', ({}, testInfo) => {
+        test.expect.soft(1+1, 'one plus one').toBe(3);
+        test.expect.soft(2*2, 'two times two').toBe(5);
+        test.expect(testInfo.errors.length, 'must be exactly two errors').toBe(2);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('Error: one plus one');
+  expect(stripAnsi(result.output)).toContain('Error: two times two');
+  expect(stripAnsi(result.output)).not.toContain('Error: must be exactly two errors');
+});

tests/playwright-test/golden.spec.ts

@@ -172,14 +172,18 @@ test('should write missing expectations locally twice and continue', async ({ ru
   expect(result.failed).toBe(1);
 
   const snapshot1OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
-  expect(result.output).toContain(`${snapshot1OutputPath} is missing in snapshots, writing actual`);
+  expect(result.output).toContain(`Error: ${snapshot1OutputPath} is missing in snapshots, writing actual`);
   expect(fs.readFileSync(snapshot1OutputPath, 'utf-8')).toBe('Hello world');
 
   const snapshot2OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot2.txt');
-  expect(result.output).toContain(`${snapshot2OutputPath} is missing in snapshots, writing actual`);
+  expect(result.output).toContain(`Error: ${snapshot2OutputPath} is missing in snapshots, writing actual`);
   expect(fs.readFileSync(snapshot2OutputPath, 'utf-8')).toBe('Hello world2');
 
   expect(result.output).toContain('Here we are!');
+
+  const stackLines = stripAnsi(result.output).split('\n').filter(line => line.includes('    at ')).filter(line => !line.includes(testInfo.outputPath()));
+  expect(result.output).toContain('a.spec.js:8');
+  expect(stackLines.length).toBe(0);
 });
 
 test('shouldn\'t write missing expectations locally for negated matcher', async ({ runInlineTest }, testInfo) => {

tests/playwright-test/reporter-github.spec.ts

@@ -89,6 +89,6 @@ test('print GitHub annotations for global error', async ({ runInlineTest }) => {
     `,
   }, { reporter: 'github' });
   const text = stripAnsi(result.output);
-  expect(text).toContain('::error ::%0AError: Oh my!%0A%0A');
+  expect(text).toContain('::error ::Error: Oh my!%0A%0A');
   expect(result.exitCode).toBe(1);
 });

utils/generate_types/overrides-test.d.ts

@@ -216,6 +216,7 @@ export interface TestInfo {
   duration: number;
   status?: TestStatus;
   error?: TestError;
+  errors: TestError[];
   stdout: (string | Buffer)[];
   stderr: (string | Buffer)[];
   snapshotSuffix: string;

utils/generate_types/overrides-testReporter.d.ts

@@ -56,6 +56,7 @@ export interface TestResult {
   duration: number;
   status: TestStatus;
   error?: TestError;
+  errors: TestError[];
   attachments: { name: string, path?: string, body?: Buffer, contentType: string }[];
   stdout: (string | Buffer)[];
   stderr: (string | Buffer)[];