chore: incremental bidi expectation update, ff expectations (#32570)

* Do not override expectations for the tests that didn't run or already
had expectations
* Sort expectations alphabetically
* Add firefox expectations

tests/bidi/expectationReporter.ts

@@ -18,7 +18,8 @@ import type {
   FullConfig, FullResult, Reporter, Suite, TestCase
 } from '@playwright/test/reporter';
 import fs from 'fs';
-import { projectExpectationPath } from './expectationUtil';
+import { parseBidiExpectations as parseExpectations, projectExpectationPath } from './expectationUtil';
+import type { TestExpectation } from './expectationUtil';
 
 type ReporterOptions = {
   rebase?: boolean;
@@ -27,6 +28,7 @@ type ReporterOptions = {
 class ExpectationReporter implements Reporter {
   private _suite: Suite;
   private _options: ReporterOptions;
+  private _pendingUpdates: Promise<void>[] = [];
 
   constructor(options: ReporterOptions) {
     this._options = options;
@@ -40,18 +42,27 @@ class ExpectationReporter implements Reporter {
     if (!this._options.rebase)
       return;
     for (const project of this._suite.suites)
-      this._updateProjectExpectations(project);
+      this._pendingUpdates.push(this._updateProjectExpectations(project));
   }
 
-  private _updateProjectExpectations(project: Suite) {
-    const results = project.allTests().map(test => {
-      const outcome = getOutcome(test);
-      const line = `${test.titlePath().slice(1).join(' › ')} [${outcome}]`;
-      return line;
-    });
+  async onExit() {
+    await Promise.all(this._pendingUpdates);
+  }
+
+  private async _updateProjectExpectations(project: Suite) {
     const outputFile = projectExpectationPath(project.title);
+    const expectations = await parseExpectations(project.title);
+    for (const test of project.allTests()) {
+      const outcome = getOutcome(test);
+      const key = test.titlePath().slice(1).join(' › ');
+      if (!expectations.has(key) || expectations.get(key) === 'unknown')
+        expectations.set(key, outcome);
+    }
+    const keys = Array.from(expectations.keys());
+    keys.sort();
+    const results = keys.map(key => `${key} [${expectations.get(key)}]`);
     console.log('Writing new expectations to', outputFile);
-    fs.writeFileSync(outputFile, results.join('\n'));
+    await fs.promises.writeFile(outputFile, results.join('\n'));
   }
 
   printsToStdio(): boolean {
@@ -59,7 +70,7 @@ class ExpectationReporter implements Reporter {
   }
 }
 
-function getOutcome(test: TestCase): 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout' {
+function getOutcome(test: TestCase): TestExpectation {
   if (test.results.length === 0)
     return 'unknown';
   if (test.results.every(r => r.status === 'timedOut'))

tests/bidi/expectationUtil.ts

@@ -18,14 +18,25 @@ import fs from 'fs';
 import path from 'path';
 import type { TestInfo } from 'playwright/test';
 
+export type TestExpectation = 'unknown' | 'flaky' | 'pass' | 'fail' | 'timeout';
+
 type ShouldSkipPredicate = (info: TestInfo) => boolean;
 
-export async function parseBidiExpectations(projectName: string): Promise<ShouldSkipPredicate> {
+export async function createSkipTestPredicate(projectName: string): Promise<ShouldSkipPredicate> {
+  const expectationsMap = await parseBidiExpectations(projectName);
+  return (info: TestInfo) => {
+    const key = [info.project.name, ...info.titlePath].join(' › ');
+    const expectation = expectationsMap.get(key);
+    return expectation === 'fail' || expectation === 'timeout';
+  };
+}
+
+export async function parseBidiExpectations(projectName: string): Promise<Map<string, TestExpectation>> {
   const filePath = projectExpectationPath(projectName);
   try {
     await fs.promises.access(filePath);
   } catch (e) {
-    return () => false;
+    return new Map();
   }
   const content = await fs.promises.readFile(filePath);
   const pairs = content.toString().split('\n').map(line => {
@@ -35,14 +46,8 @@ export async function parseBidiExpectations(projectName: string): Promise<Should
       return undefined;
     }
     return [match.groups!.titlePath, match.groups!.expectation];
-  }).filter(Boolean) as [string, string][];
-  const expectationsMap = new Map(pairs);
-
-  return (info: TestInfo) => {
-    const key = [info.project.name, ...info.titlePath].join(' › ');
-    const expectation = expectationsMap.get(key);
-    return expectation === 'fail' || expectation === 'timeout';
-  };
+  }).filter(Boolean) as [string, TestExpectation][];
+  return new Map(pairs);
 }
 
 export function projectExpectationPath(project: string): string {

tests/config/browserTest.ts

@@ -24,7 +24,7 @@ import { baseTest } from './baseTest';
 import { type RemoteServerOptions, type PlaywrightServer, RunServer, RemoteServer } from './remoteServer';
 import type { Log } from '../../packages/trace/src/har';
 import { parseHar } from '../config/utils';
-import { parseBidiExpectations as parseBidiProjectExpectations } from '../bidi/expectationUtil';
+import { createSkipTestPredicate } from '../bidi/expectationUtil';
 import type { TestInfo } from '@playwright/test';
 
 export type BrowserTestWorkerFixtures = PageWorkerFixtures & {
@@ -172,7 +172,7 @@ const test = baseTest.extend<BrowserTestTestFixtures, BrowserTestWorkerFixtures>
   },
 
   bidiTestSkipPredicate: [async ({ }, run) => {
-    const filter = await parseBidiProjectExpectations(test.info().project.name);
+    const filter = await createSkipTestPredicate(test.info().project.name);
     await run(filter);
   }, { scope: 'worker' }],