From 7f141b2c4202a2ea3754bbf8e8d4240e89f41641 Mon Sep 17 00:00:00 2001
From: Pengoose <pengoose_dev@naver.com>
Date: Fri, 27 Dec 2024 18:54:16 +0900
Subject: [PATCH] feat: expect(locator).toHaveAccessibleErrorMessage (#33904)

---
 docs/src/api/class-locatorassertions.md       |  50 +++++++
 .../src/server/injected/injectedScript.ts     |   4 +-
 .../src/server/injected/roleUtils.ts          |  56 ++++++++
 packages/playwright/src/matchers/expect.ts    |   2 +
 packages/playwright/src/matchers/matchers.ts  |  12 ++
 packages/playwright/types/test.d.ts           |  28 ++++
 tests/page/expect-misc.spec.ts                | 130 ++++++++++++++++++
 7 files changed, 281 insertions(+), 1 deletion(-)

diff --git a/docs/src/api/class-locatorassertions.md b/docs/src/api/class-locatorassertions.md
index 7e61e1b3a5..c2adf3afc5 100644
--- a/docs/src/api/class-locatorassertions.md
+++ b/docs/src/api/class-locatorassertions.md
@@ -1217,6 +1217,56 @@ Expected accessible description.
 * since: v1.44
 
 
+## async method: LocatorAssertions.toHaveAccessibleErrorMessage
+* since: v1.50
+* langs:
+  - alias-java: hasAccessibleErrorMessage
+
+Ensures the [Locator] points to an element with a given [aria errormessage](https://w3c.github.io/aria/#aria-errormessage).
+
+**Usage**
+
+```js
+const locator = page.getByTestId('username-input');
+await expect(locator).toHaveAccessibleErrorMessage('Username is required.');
+```
+
+```java
+Locator locator = page.getByTestId("username-input");
+assertThat(locator).hasAccessibleErrorMessage("Username is required.");
+```
+
+```python async
+locator = page.get_by_test_id("username-input")
+await expect(locator).to_have_accessible_error_message("Username is required.")
+```
+
+```python sync
+locator = page.get_by_test_id("username-input")
+expect(locator).to_have_accessible_error_message("Username is required.")
+```
+
+```csharp
+var locator = Page.GetByTestId("username-input");
+await Expect(locator).ToHaveAccessibleErrorMessageAsync("Username is required.");
+```
+
+### param: LocatorAssertions.toHaveAccessibleErrorMessage.errorMessage
+* since: v1.50
+- `errorMessage` <[string]|[RegExp]>
+
+Expected accessible error message.
+
+### option: LocatorAssertions.toHaveAccessibleErrorMessage.timeout = %%-js-assertions-timeout-%%
+* since: v1.50
+
+### option: LocatorAssertions.toHaveAccessibleErrorMessage.timeout = %%-csharp-java-python-assertions-timeout-%%
+* since: v1.50
+
+### option: LocatorAssertions.toHaveAccessibleErrorMessage.ignoreCase = %%-assertions-ignore-case-%%
+* since: v1.50
+
+
 ## async method: LocatorAssertions.toHaveAccessibleName
 * since: v1.44
 * langs:
diff --git a/packages/playwright-core/src/server/injected/injectedScript.ts b/packages/playwright-core/src/server/injected/injectedScript.ts
index a1ffdf893c..21fcca69a4 100644
--- a/packages/playwright-core/src/server/injected/injectedScript.ts
+++ b/packages/playwright-core/src/server/injected/injectedScript.ts
@@ -29,7 +29,7 @@ import type { CSSComplexSelectorList } from '../../utils/isomorphic/cssParser';
 import { generateSelector, type GenerateSelectorOptions } from './selectorGenerator';
 import type * as channels from '@protocol/channels';
 import { Highlight } from './highlight';
-import { getChecked, getAriaDisabled, getAriaRole, getElementAccessibleName, getElementAccessibleDescription, getReadonly } from './roleUtils';
+import { getChecked, getAriaDisabled, getAriaRole, getElementAccessibleName, getElementAccessibleDescription, getReadonly, getElementAccessibleErrorMessage } from './roleUtils';
 import { kLayoutSelectorNames, type LayoutSelectorName, layoutSelectorScore } from './layoutSelectorUtils';
 import { asLocator } from '../../utils/isomorphic/locatorGenerators';
 import type { Language } from '../../utils/isomorphic/locatorGenerators';
@@ -1321,6 +1321,8 @@ export class InjectedScript {
         received = getElementAccessibleName(element, false /* includeHidden */);
       } else if (expression === 'to.have.accessible.description') {
         received = getElementAccessibleDescription(element, false /* includeHidden */);
+      } else if (expression === 'to.have.accessible.error.message') {
+        received = getElementAccessibleErrorMessage(element);
       } else if (expression === 'to.have.role') {
         received = getAriaRole(element) || '';
       } else if (expression === 'to.have.title') {
diff --git a/packages/playwright-core/src/server/injected/roleUtils.ts b/packages/playwright-core/src/server/injected/roleUtils.ts
index cc9e6a70d0..f74c893c1b 100644
--- a/packages/playwright-core/src/server/injected/roleUtils.ts
+++ b/packages/playwright-core/src/server/injected/roleUtils.ts
@@ -461,6 +461,59 @@ export function getElementAccessibleDescription(element: Element, includeHidden:
   return accessibleDescription;
 }
 
+// https://www.w3.org/TR/wai-aria-1.2/#aria-invalid
+const kAriaInvalidRoles = ['application', 'checkbox', 'combobox', 'gridcell', 'listbox', 'radiogroup', 'slider', 'spinbutton', 'textbox', 'tree', 'columnheader', 'rowheader', 'searchbox', 'switch', 'treegrid'];
+
+function getAriaInvalid(element: Element): 'false' | 'true' | 'grammar' | 'spelling' {
+  const role = getAriaRole(element) || '';
+  if (!role || !kAriaInvalidRoles.includes(role))
+    return 'false';
+  const ariaInvalid = element.getAttribute('aria-invalid');
+  if (!ariaInvalid || ariaInvalid.trim() === '' || ariaInvalid.toLocaleLowerCase() === 'false')
+    return 'false';
+  if (ariaInvalid === 'true' || ariaInvalid === 'grammar' || ariaInvalid === 'spelling')
+    return ariaInvalid;
+  return 'true';
+}
+
+function getValidityInvalid(element: Element) {
+  if ('validity' in element){
+    const validity = element.validity as ValidityState | undefined;
+    return validity?.valid === false;
+  }
+  return false;
+}
+
+export function getElementAccessibleErrorMessage(element: Element): string {
+  // SPEC: https://w3c.github.io/aria/#aria-errormessage
+  //
+  // TODO: support https://developer.mozilla.org/en-US/docs/Web/API/HTMLInputElement/validationMessage
+  const cache = cacheAccessibleErrorMessage;
+  let accessibleErrorMessage = cacheAccessibleErrorMessage?.get(element);
+
+  if (accessibleErrorMessage === undefined) {
+    accessibleErrorMessage = '';
+
+    const isAriaInvalid = getAriaInvalid(element) !== 'false';
+    const isValidityInvalid = getValidityInvalid(element);
+    if (isAriaInvalid || isValidityInvalid) {
+      const errorMessageId = element.getAttribute('aria-errormessage');
+      const errorMessages = getIdRefs(element, errorMessageId);
+      // Ideally, this should be a separate "embeddedInErrorMessage", but it would follow the exact same rules.
+      // Relevant vague spec: https://w3c.github.io/core-aam/#ariaErrorMessage.
+      const parts = errorMessages.map(errorMessage => asFlatString(
+          getTextAlternativeInternal(errorMessage, {
+            visitedElements: new Set(),
+            embeddedInDescribedBy: { element: errorMessage, hidden: isElementHiddenForAria(errorMessage) },
+          })
+      ));
+      accessibleErrorMessage = parts.join(' ').trim();
+    }
+    cache?.set(element, accessibleErrorMessage);
+  }
+  return accessibleErrorMessage;
+}
+
 type AccessibleNameOptions = {
   visitedElements: Set<Element>,
   includeHidden?: boolean,
@@ -972,6 +1025,7 @@ let cacheAccessibleName: Map<Element, string> | undefined;
 let cacheAccessibleNameHidden: Map<Element, string> | undefined;
 let cacheAccessibleDescription: Map<Element, string> | undefined;
 let cacheAccessibleDescriptionHidden: Map<Element, string> | undefined;
+let cacheAccessibleErrorMessage: Map<Element, string> | undefined;
 let cacheIsHidden: Map<Element, boolean> | undefined;
 let cachePseudoContentBefore: Map<Element, string> | undefined;
 let cachePseudoContentAfter: Map<Element, string> | undefined;
@@ -983,6 +1037,7 @@ export function beginAriaCaches() {
   cacheAccessibleNameHidden ??= new Map();
   cacheAccessibleDescription ??= new Map();
   cacheAccessibleDescriptionHidden ??= new Map();
+  cacheAccessibleErrorMessage ??= new Map();
   cacheIsHidden ??= new Map();
   cachePseudoContentBefore ??= new Map();
   cachePseudoContentAfter ??= new Map();
@@ -994,6 +1049,7 @@ export function endAriaCaches() {
     cacheAccessibleNameHidden = undefined;
     cacheAccessibleDescription = undefined;
     cacheAccessibleDescriptionHidden = undefined;
+    cacheAccessibleErrorMessage = undefined;
     cacheIsHidden = undefined;
     cachePseudoContentBefore = undefined;
     cachePseudoContentAfter = undefined;
diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts
index 0bd116e7a1..d4c3287d33 100644
--- a/packages/playwright/src/matchers/expect.ts
+++ b/packages/playwright/src/matchers/expect.ts
@@ -35,6 +35,7 @@ import {
   toContainText,
   toHaveAccessibleDescription,
   toHaveAccessibleName,
+  toHaveAccessibleErrorMessage,
   toHaveAttribute,
   toHaveClass,
   toHaveCount,
@@ -224,6 +225,7 @@ const customAsyncMatchers = {
   toContainText,
   toHaveAccessibleDescription,
   toHaveAccessibleName,
+  toHaveAccessibleErrorMessage,
   toHaveAttribute,
   toHaveClass,
   toHaveCount,
diff --git a/packages/playwright/src/matchers/matchers.ts b/packages/playwright/src/matchers/matchers.ts
index 8a8089e91e..3962c0cae9 100644
--- a/packages/playwright/src/matchers/matchers.ts
+++ b/packages/playwright/src/matchers/matchers.ts
@@ -205,6 +205,18 @@ export function toHaveAccessibleName(
   }
 }
 
+export function toHaveAccessibleErrorMessage(
+  this: ExpectMatcherState,
+  locator: LocatorEx,
+  expected: string | RegExp,
+  options?: { timeout?: number; ignoreCase?: boolean },
+) {
+  return toMatchText.call(this, 'toHaveAccessibleErrorMessage', locator, 'Locator', async (isNot, timeout) => {
+    const expectedText = serializeExpectedTextValues([expected], { ignoreCase: options?.ignoreCase, normalizeWhiteSpace: true });
+    return await locator._expect('to.have.accessible.error.message', { expectedText: expectedText, isNot, timeout });
+  }, expected, options);
+}
+
 export function toHaveAttribute(
   this: ExpectMatcherState,
   locator: LocatorEx,
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 85e3d37847..7c9cae5415 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -8112,6 +8112,34 @@ interface LocatorAssertions {
     timeout?: number;
   }): Promise<void>;
 
+  /**
+   * Ensures the [Locator](https://playwright.dev/docs/api/class-locator) points to an element with a given
+   * [aria errormessage](https://w3c.github.io/aria/#aria-errormessage).
+   *
+   * **Usage**
+   *
+   * ```js
+   * const locator = page.getByTestId('username-input');
+   * await expect(locator).toHaveAccessibleErrorMessage('Username is required.');
+   * ```
+   *
+   * @param errorMessage Expected accessible error message.
+   * @param options
+   */
+  toHaveAccessibleErrorMessage(errorMessage: string|RegExp, options?: {
+    /**
+     * Whether to perform case-insensitive match.
+     * [`ignoreCase`](https://playwright.dev/docs/api/class-locatorassertions#locator-assertions-to-have-accessible-error-message-option-ignore-case)
+     * option takes precedence over the corresponding regular expression flag if specified.
+     */
+    ignoreCase?: boolean;
+
+    /**
+     * Time to retry the assertion for in milliseconds. Defaults to `timeout` in `TestConfig.expect`.
+     */
+    timeout?: number;
+  }): Promise<void>;
+
   /**
    * Ensures the [Locator](https://playwright.dev/docs/api/class-locator) points to an element with a given
    * [accessible name](https://w3c.github.io/accname/#dfn-accessible-name).
diff --git a/tests/page/expect-misc.spec.ts b/tests/page/expect-misc.spec.ts
index 0ab5707a62..a1fb6637b1 100644
--- a/tests/page/expect-misc.spec.ts
+++ b/tests/page/expect-misc.spec.ts
@@ -491,6 +491,136 @@ test('toHaveAccessibleDescription', async ({ page }) => {
   await expect(page.locator('div')).toHaveAccessibleDescription('foo bar baz');
 });
 
+test('toHaveAccessibleErrorMessage', async ({ page }) => {
+  await page.setContent(`
+    <form>
+      <input role="textbox" aria-invalid="true" aria-errormessage="error-message" />
+      <div id="error-message">Hello</div>
+      <div id="irrelevant-error">This should not be considered.</div>
+    </form>
+  `);
+
+  const locator = page.locator('input[role="textbox"]');
+  await expect(locator).toHaveAccessibleErrorMessage('Hello');
+  await expect(locator).not.toHaveAccessibleErrorMessage('hello');
+  await expect(locator).toHaveAccessibleErrorMessage('hello', { ignoreCase: true });
+  await expect(locator).toHaveAccessibleErrorMessage(/ell\w/);
+  await expect(locator).not.toHaveAccessibleErrorMessage(/hello/);
+  await expect(locator).toHaveAccessibleErrorMessage(/hello/, { ignoreCase: true });
+  await expect(locator).not.toHaveAccessibleErrorMessage('This should not be considered.');
+});
+
+test('toHaveAccessibleErrorMessage should handle multiple aria-errormessage references', async ({ page }) => {
+  await page.setContent(`
+    <form>
+      <input role="textbox" aria-invalid="true" aria-errormessage="error1 error2" />
+      <div id="error1">First error message.</div>
+      <div id="error2">Second error message.</div>
+      <div id="irrelevant-error">This should not be considered.</div>
+    </form>
+  `);
+
+  const locator = page.locator('input[role="textbox"]');
+
+  await expect(locator).toHaveAccessibleErrorMessage('First error message. Second error message.');
+  await expect(locator).toHaveAccessibleErrorMessage(/first error message./i);
+  await expect(locator).toHaveAccessibleErrorMessage(/second error message./i);
+  await expect(locator).not.toHaveAccessibleErrorMessage(/This should not be considered./i);
+});
+
+test.describe('toHaveAccessibleErrorMessage should handle aria-invalid attribute', () => {
+  const errorMessageText = 'Error message';
+
+  async function setupPage(page, ariaInvalidValue: string | null) {
+    const ariaInvalidAttr = ariaInvalidValue === null ? '' : `aria-invalid="${ariaInvalidValue}"`;
+    await page.setContent(`
+        <form>
+          <input id="node" role="textbox" ${ariaInvalidAttr} aria-errormessage="error-msg" />
+          <div id="error-msg">${errorMessageText}</div>
+        </form>
+      `);
+    return page.locator('#node');
+  }
+
+  test.describe('evaluated in false', () => {
+    test('no aria-invalid attribute', async ({ page }) => {
+      const locator = await setupPage(page, null);
+      await expect(locator).not.toHaveAccessibleErrorMessage(errorMessageText);
+    });
+    test('aria-invalid="false"', async ({ page }) => {
+      const locator = await setupPage(page, 'false');
+      await expect(locator).not.toHaveAccessibleErrorMessage(errorMessageText);
+    });
+    test('aria-invalid="" (empty string)', async ({ page }) => {
+      const locator = await setupPage(page, '');
+      await expect(locator).not.toHaveAccessibleErrorMessage(errorMessageText);
+    });
+  });
+  test.describe('evaluated in true', () => {
+    test('aria-invalid="true"', async ({ page }) => {
+      const locator = await setupPage(page, 'true');
+      await expect(locator).toHaveAccessibleErrorMessage(errorMessageText);
+    });
+    test('aria-invalid="foo" (unrecognized value)', async ({ page }) => {
+      const locator = await setupPage(page, 'foo');
+      await expect(locator).toHaveAccessibleErrorMessage(errorMessageText);
+    });
+  });
+});
+
+test.describe('toHaveAccessibleErrorMessage should handle validity state with aria-invalid', () => {
+  const errorMessageText = 'Error message';
+
+  test('should show error message when validity is false and aria-invalid is true', async ({ page }) => {
+    await page.setContent(`
+      <form>
+        <input id="node" role="textbox" type="number" min="1" max="100" aria-invalid="true" aria-errormessage="error-msg" />
+        <div id="error-msg">${errorMessageText}</div>
+      </form>
+    `);
+    const locator = page.locator('#node');
+    await locator.fill('101');
+    await expect(locator).toHaveAccessibleErrorMessage(errorMessageText);
+  });
+
+  test('should show error message when validity is true and aria-invalid is true', async ({ page }) => {
+    await page.setContent(`
+      <form>
+        <input id="node" role="textbox" type="number" min="1" max="100" aria-invalid="true" aria-errormessage="error-msg" />
+        <div id="error-msg">${errorMessageText}</div>
+      </form>
+    `);
+    const locator = page.locator('#node');
+    await locator.fill('99');
+    await expect(locator).toHaveAccessibleErrorMessage(errorMessageText);
+  });
+
+  test('should show error message when validity is false and aria-invalid is false', async ({ page }) => {
+    await page.setContent(`
+      <form>
+        <input id="node" role="textbox" type="number" min="1" max="100" aria-invalid="false" aria-errormessage="error-msg" />
+        <div id="error-msg">${errorMessageText}</div>
+      </form>
+    `);
+    const locator = page.locator('#node');
+    await locator.fill('101');
+    await expect(locator).toHaveAccessibleErrorMessage(errorMessageText);
+  });
+
+  test('should not show error message when validity is true and aria-invalid is false', async ({ page }) => {
+    await page.setContent(`
+      <form>
+        <input id="node" role="textbox" type="number" min="1" max="100" aria-invalid="false" aria-errormessage="error-msg" />
+        <div id="error-msg">${errorMessageText}</div>
+      </form>
+    `);
+    const locator = page.locator('#node');
+    await locator.fill('99');
+    await expect(locator).not.toHaveAccessibleErrorMessage(errorMessageText);
+  });
+});
+
+
 test('toHaveRole', async ({ page }) => {
   await page.setContent(`<div role="button">Button!</div>`);
   await expect(page.locator('div')).toHaveRole('button');