diff --git a/tests/playwright-test/warnings.spec.ts b/tests/playwright-test/warnings.spec.ts
index d9ed42eb5d..ac69ac7d31 100644
--- a/tests/playwright-test/warnings.spec.ts
+++ b/tests/playwright-test/warnings.spec.ts
@@ -14,15 +14,18 @@
  * limitations under the License.
  */
 
+import { JSONReport } from 'packages/playwright-test/reporter';
 import { test, expect } from './playwright-test-fixtures';
 
 const warningSnippet = 'Some async calls were not awaited';
 
 test.describe.configure({ mode: 'parallel' });
 
+const getWarnings = (report: JSONReport) => report.suites.flatMap(s => s.specs).flatMap(s => s.tests).flatMap(t => t.annotations).filter(a => a.type === 'warning');
+
 test.describe('await', () => {
   test('should not care about non-API promises', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test } from '@playwright/test';
         test('test', () => {
@@ -31,11 +34,12 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(0);
-    expect(stdout).not.toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(0);
   });
 
   test('should warn about missing await on expects when failing', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('custom test name', async ({ page }) => {
@@ -44,13 +48,14 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(1);
-    expect(stdout).toContain(warningSnippet);
-    expect(stdout).toContain('the test');
-    expect(stdout).toContain('custom test name');
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(1);
+    expect(warnings[0].description).toContain(warningSnippet);
+    expect(warnings[0].description).toContain('the test');
   });
 
   test('should warn about missing await on expects when passing', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('test', async ({ page }) => {
@@ -60,11 +65,13 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(0);
-    expect(stdout).toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(1);
+    expect(warnings[0].description).toContain(warningSnippet);
   });
 
   test('should not warn when not missing await on expects when failing', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('test', async ({ page }) => {
@@ -73,11 +80,12 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(1);
-    expect(stdout).not.toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(0);
   });
 
   test('should not warn when not missing await on expects when passing', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('test', async ({ page }) => {
@@ -87,11 +95,12 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(0);
-    expect(stdout).not.toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(0);
   });
 
   test('should not warn when using then on expects when passing', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('test', async ({ page }) => {
@@ -101,11 +110,12 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(0);
-    expect(stdout).not.toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(0);
   });
 
   test('should warn about missing await on reject', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('test', async ({ page }) => {
@@ -114,11 +124,13 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(0);
-    expect(stdout).toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(1);
+    expect(warnings[0].description).toContain(warningSnippet);
   });
 
   test('should warn about missing await on reject.not', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('test', async ({ page }) => {
@@ -127,11 +139,13 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(1);
-    expect(stdout).toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(1);
+    expect(warnings[0].description).toContain(warningSnippet);
   });
 
   test('should warn about missing await on test.step', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('test', async ({ page }) => {
@@ -142,11 +156,13 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(0);
-    expect(stdout).toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(1);
+    expect(warnings[0].description).toContain(warningSnippet);
   });
 
   test('should not warn when not missing await on test.step', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('test', async ({ page }) => {
@@ -157,11 +173,12 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(0);
-    expect(stdout).not.toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(0);
   });
 
   test('should warn about missing await on test.step.skip', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test('test', async ({ page }) => {
@@ -172,7 +189,9 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(0);
-    expect(stdout).toContain(warningSnippet);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(1);
+    expect(warnings[0].description).toContain(warningSnippet);
   });
 
   test('traced promise should be instanceof Promise', async ({ runInlineTest }) => {
@@ -193,7 +212,7 @@ test.describe('await', () => {
     const group = ['beforeAll', 'beforeEach'];
     for (const hook of group) {
       await test.step(hook, async () => {
-        const { exitCode, stdout } = await runInlineTest({
+        const { exitCode, report } = await runInlineTest({
           'a.test.ts': `
             import { test, expect } from '@playwright/test';
             let page;
@@ -209,8 +228,10 @@ test.describe('await', () => {
         });
 
         expect(exitCode).toBe(0);
-        expect(stdout).toContain(warningSnippet);
-        expect(stdout).toContain(`${group[0]}/${group[1]} hooks`);
+        const warnings = getWarnings(report);
+        expect(warnings.length).toEqual(1);
+        expect(warnings[0].description).toContain(warningSnippet);
+        expect(warnings[0].description).toContain(`${group[0]}/${group[1]} hooks`);
       });
     }
   });
@@ -219,7 +240,7 @@ test.describe('await', () => {
     const group = ['afterAll', 'afterEach'];
     for (const hook of group) {
       test(hook, async ({ runInlineTest }) => {
-        const { exitCode, stdout } = await runInlineTest({
+        const { exitCode, report } = await runInlineTest({
           'a.test.ts': `
             import { test, expect } from '@playwright/test';
             let page;
@@ -233,14 +254,16 @@ test.describe('await', () => {
         });
 
         expect(exitCode).toBe(0);
-        expect(stdout).toContain(warningSnippet);
-        expect(stdout).toContain(`${group[0]}/${group[1]} hooks`);
+        const warnings = getWarnings(report);
+        expect(warnings.length).toEqual(1);
+        expect(warnings[0].description).toContain(warningSnippet);
+        expect(warnings[0].description).toContain(`${group[0]}/${group[1]} hooks`);
       });
     }
   });
 
   test('should warn about missing await across hooks and test', async ({ runInlineTest }) => {
-    const { exitCode, stdout } = await runInlineTest({
+    const { exitCode, report } = await runInlineTest({
       'a.test.ts': `
         import { test, expect } from '@playwright/test';
         test.beforeAll(async () => {
@@ -255,8 +278,10 @@ test.describe('await', () => {
       `
     });
     expect(exitCode).toBe(0);
-    expect(stdout).toContain(`${warningSnippet} by the end of beforeAll/beforeEach hooks.`);
-    expect(stdout).toContain(`${warningSnippet} by the end of the test.`);
-    expect(stdout).toContain(`${warningSnippet} by the end of afterAll/afterEach hooks.`);
+    const warnings = getWarnings(report);
+    expect(warnings.length).toEqual(3);
+    expect(warnings[0].description).toContain(`${warningSnippet} by the end of beforeAll/beforeEach hooks.`);
+    expect(warnings[1].description).toContain(`${warningSnippet} by the end of the test.`);
+    expect(warnings[2].description).toContain(`${warningSnippet} by the end of afterAll/afterEach hooks.`);
   });
 });