chore: bring in folio source (#6923)

- Source now lives at `src/test`.
- Former folio tests live at `tests/playwright-test`.
- We use `src/test/internal.ts` that exposes base test without
  Playwright fixtures for most tests (to avoid modifications for now).
- Test types live in `types/testFoo.d.ts`.
- Stable test runner is installed to `tests/config/test-runner` during `npm install`.
- All deps including test-only are now listed in `package.json`.
  Non-test deps must also be listed in `build_package.js` to get included.

.github/workflows/tests_primary.yml

@@ -47,3 +47,20 @@ jobs:
         name: ${{ matrix.browser }}-${{ matrix.os }}-test-results
         path: test-results
 
+  test_test_runner:
+    name: Test Runner
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: 12
+    - run: npm ci
+      env:
+        DEBUG: pw:install
+    - run: npm run build
+    - run: npm run ttest

install-from-github.js

@@ -18,6 +18,17 @@
 // This file is only run when someone installs via the github repo
 
 const {execSync} = require('child_process');
+const path = require('path');
+
+console.log(`Updating test runner...`);
+try {
+  execSync('npm ci --save=false --fund=false --audit=false', {
+    stdio: ['inherit', 'inherit', 'inherit'],
+    cwd: path.join(__dirname, 'tests', 'config', 'test-runner'),
+  });
+} catch (e) {
+  process.exit(1);
+}
 
 console.log(`Rebuilding installer...`);
 try {

package.json

@@ -9,12 +9,14 @@
     "node": ">=12"
   },
   "scripts": {
-    "ctest": "folio --config=tests/config/default.config.ts --project=chromium",
+    "basetest": "node ./tests/config/test-runner/node_modules/@playwright/test/lib/cli/cli.js test",
-    "ftest": "folio --config=tests/config/default.config.ts --project=firefox",
+    "ctest": "npm run basetest -- --config=tests/config/default.config.ts --project=chromium",
-    "wtest": "folio --config=tests/config/default.config.ts --project=webkit",
+    "ftest": "npm run basetest -- --config=tests/config/default.config.ts --project=firefox",
-    "atest": "folio --config=tests/config/android.config.ts",
+    "wtest": "npm run basetest -- --config=tests/config/default.config.ts --project=webkit",
-    "etest": "folio --config=tests/config/electron.config.ts",
+    "atest": "npm run basetest -- --config=tests/config/android.config.ts",
-    "test": "folio --config=tests/config/default.config.ts",
+    "etest": "npm run basetest -- --config=tests/config/electron.config.ts",
+    "ttest": "npm run basetest -- --config=tests/playwright-test/playwright-test.config.ts",
+    "test": "npm run basetest -- --config=tests/config/default.config.ts",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext ts . || eslint --ext ts .",
     "tsc": "tsc -p .",
     "tsc-installer": "tsc -p ./src/install/tsconfig.json",
@@ -36,7 +38,32 @@
   "bin": {
     "playwright": "./lib/cli/cli.js"
   },
+  "DEPS-NOTE": "Any non-test dependency must be added to the build_package.js script as well",
   "dependencies": {
+    "@babel/code-frame": "^7.12.13",
+    "@babel/core": "^7.14.0",
+    "@babel/plugin-proposal-class-properties": "^7.13.0",
+    "@babel/plugin-proposal-dynamic-import": "^7.13.8",
+    "@babel/plugin-proposal-export-namespace-from": "^7.12.13",
+    "@babel/plugin-proposal-logical-assignment-operators": "^7.13.8",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8",
+    "@babel/plugin-proposal-numeric-separator": "^7.12.13",
+    "@babel/plugin-proposal-optional-chaining": "^7.13.12",
+    "@babel/plugin-proposal-private-methods": "^7.13.0",
+    "@babel/plugin-proposal-private-property-in-object": "^7.14.0",
+    "@babel/plugin-syntax-async-generators": "^7.8.4",
+    "@babel/plugin-syntax-json-strings": "^7.8.3",
+    "@babel/plugin-syntax-object-rest-spread": "^7.8.3",
+    "@babel/plugin-syntax-optional-catch-binding": "^7.8.3",
+    "@babel/plugin-transform-modules-commonjs": "^7.14.0",
+    "@babel/preset-typescript": "^7.13.0",
+    "colors": "^1.4.0",
+    "expect": "^26.4.2",
+    "minimatch": "^3.0.3",
+    "ms": "^2.1.2",
+    "pirates": "^4.0.1",
+    "pixelmatch": "^5.2.1",
+    "source-map-support": "^0.4.18",
     "commander": "^6.1.0",
     "debug": "^4.1.1",
     "extract-zip": "^2.0.1",
@@ -53,10 +80,14 @@
     "yazl": "^2.5.1"
   },
   "devDependencies": {
+    "@types/babel__code-frame": "^7.0.2",
+    "@types/babel__core": "^7.1.14",
     "@types/debug": "^4.1.5",
     "@types/extract-zip": "^1.6.2",
     "@types/mime": "^2.0.3",
+    "@types/minimatch": "^3.0.3",
     "@types/node": "^10.17.28",
+    "@types/pixelmatch": "^5.2.1",
     "@types/pngjs": "^3.4.2",
     "@types/progress": "^2.0.3",
     "@types/proper-lockfile": "^4.1.1",
@@ -65,8 +96,10 @@
     "@types/react-dom": "^17.0.0",
     "@types/resize-observer-browser": "^0.1.4",
     "@types/rimraf": "^3.0.0",
+    "@types/source-map-support": "^0.4.2",
     "@types/webpack": "^4.41.25",
     "@types/ws": "7.2.6",
+    "@types/xml2js": "^0.4.5",
     "@types/yazl": "^2.4.2",
     "@typescript-eslint/eslint-plugin": "^4.25.0",
     "@typescript-eslint/parser": "^4.25.0",
@@ -80,7 +113,6 @@
     "eslint-plugin-notice": "^0.9.10",
     "eslint-plugin-react-hooks": "^4.2.0",
     "file-loader": "^6.1.0",
-    "folio": "=0.4.0-alpha28",
     "formidable": "^1.2.2",
     "html-webpack-plugin": "^4.4.1",
     "ncp": "^2.0.0",
@@ -90,9 +122,10 @@
     "socksv5": "0.0.6",
     "style-loader": "^1.2.1",
     "ts-loader": "^8.0.3",
-    "typescript": "^4.0.2",
+    "typescript": "=4.2.4",
     "webpack": "^4.44.2",
     "webpack-cli": "^3.3.12",
+    "xml2js": "^0.4.23",
     "yaml": "^1.10.0"
   }
 }

packages/build_package.js

@@ -61,10 +61,27 @@ const PACKAGES = {
   'playwright-chromium': {
     description: 'A high-level API to automate Chromium',
     browsers: ['chromium', 'ffmpeg'],
-    files: [...PLAYWRIGHT_CORE_FILES],
+    files: PLAYWRIGHT_CORE_FILES,
   },
 };
 
+const DEPENDENCIES = [
+  'commander',
+  'debug',
+  'extract-zip',
+  'https-proxy-agent',
+  'jpeg-js',
+  'mime',
+  'pngjs',
+  'progress',
+  'proper-lockfile',
+  'proxy-from-env',
+  'rimraf',
+  'stack-utils',
+  'ws',
+  'yazl',
+];
+
 // 1. Parse CLI arguments
 const args = process.argv.slice(2);
 if (args.some(arg => arg === '--help')) {
@@ -121,9 +138,10 @@ if (!args.some(arg => arg === '--no-cleanup')) {
 
   // 4. Generate package.json
   const pwInternalJSON = require(path.join(ROOT_PATH, 'package.json'));
-  const dependencies = { ...pwInternalJSON.dependencies };
+  const depNames = packageName === 'playwright-test' ? Object.keys(pwInternalJSON.dependencies) : DEPENDENCIES;
-  if (packageName === 'playwright-test')
+  const dependencies = {};
-    dependencies.folio = pwInternalJSON.devDependencies.folio;
+  for (const dep of depNames)
+    dependencies[dep] =  pwInternalJSON.dependencies[dep];
   await writeToPackage('package.json', JSON.stringify({
     name: package.name || packageName,
     version: pwInternalJSON.version,

packages/playwright-test/index.js

@@ -16,5 +16,5 @@
 
 module.exports = {
   ...require('./lib/inprocess'),
-  ...require('./lib/cli/fixtures')
+  ...require('./lib/test/index')
 };

src/cli/cli.ts

@@ -35,16 +35,16 @@ import { BrowserContextOptions, LaunchOptions } from '../client/types';
 import { spawn } from 'child_process';
 import { installDeps } from '../install/installDeps';
 import { allBrowserNames, BrowserName } from '../utils/registry';
-import { addTestCommand } from './testRunner';
 import * as utils from '../utils/utils';
 
 const SCRIPTS_DIRECTORY = path.join(__dirname, '..', '..', 'bin');
 
 type BrowserChannel = 'chrome-beta'|'chrome';
 const allBrowserChannels: Set<BrowserChannel> = new Set(['chrome-beta', 'chrome']);
+const packageJSON = require('../../package.json');
 
 program
-    .version('Version ' + require('../../package.json').version)
+    .version('Version ' + packageJSON.version)
     .name(process.env.PW_CLI_NAME || 'npx playwright');
 
 commandWithOpenOptions('open [url]', 'open page in browser specified via -b, --browser', [])
@@ -226,8 +226,19 @@ program
       console.log('  $ show-trace trace/directory');
     });
 
-if (!process.env.PW_CLI_TARGET_LANG)
+if (!process.env.PW_CLI_TARGET_LANG) {
-  addTestCommand(program);
+  if (packageJSON.name === '@playwright/test' || process.env.PWTEST_CLI_ALLOW_TEST_COMMAND) {
+    require('../test/cli').addTestCommand(program);
+  } else {
+    const command = program.command('test');
+    command.description('Run tests with Playwright Test. Available in @playwright/test package.');
+    command.action(async (args, opts) => {
+      console.error('Please install @playwright/test package to use Playwright Test.');
+      console.error('  npm install -D @playwright/test');
+      process.exit(1);
+    });
+  }
+}
 
 if (process.argv[2] === 'run-driver')
   runDriver();

src/test/cli.ts

@@ -19,9 +19,8 @@
 import * as commander from 'commander';
 import * as fs from 'fs';
 import * as path from 'path';
-import type { Config } from 'folio';
+import type { Config } from './types';
-
+import { Runner } from './runner';
-type RunnerType = typeof import('folio/out/runner').Runner;
 
 const defaultTimeout = 30000;
 const defaultReporter = process.env.CI ? 'dot' : 'list';
@@ -37,14 +36,6 @@ const defaultConfig: Config = {
 };
 
 export function addTestCommand(program: commander.CommanderStatic) {
-  let Runner: RunnerType;
-  try {
-    Runner = require('folio/out/runner').Runner as RunnerType;
-  } catch (e) {
-    addStubTestCommand(program);
-    return;
-  }
-
   const command = program.command('test [test-filter...]');
   command.description('Run tests with Playwright Test');
   command.option('--browser <browser>', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`);
@@ -68,7 +59,7 @@ export function addTestCommand(program: commander.CommanderStatic) {
   command.option('-x', `Stop after the first failure`);
   command.action(async (args, opts) => {
     try {
-      await runTests(Runner, args, opts);
+      await runTests(args, opts);
     } catch (e) {
       console.error(e.toString());
       process.exit(1);
@@ -86,7 +77,7 @@ export function addTestCommand(program: commander.CommanderStatic) {
   });
 }
 
-async function runTests(Runner: RunnerType, args: string[], opts: { [key: string]: any }) {
+async function runTests(args: string[], opts: { [key: string]: any }) {
   const browserOpt = opts.browser ? opts.browser.toLowerCase() : 'chromium';
   if (!['all', 'chromium', 'firefox', 'webkit'].includes(browserOpt))
     throw new Error(`Unsupported browser "${opts.browser}", must be one of "all", "chromium", "firefox" or "webkit"`);
@@ -135,11 +126,6 @@ async function runTests(Runner: RunnerType, args: string[], opts: { [key: string
     throw new Error(`Configuration file not found. Run "npx playwright test --help" for more information.`);
   }
 
-  process.env.FOLIO_JUNIT_OUTPUT_NAME = process.env.PLAYWRIGHT_JUNIT_OUTPUT_NAME;
-  process.env.FOLIO_JUNIT_SUITE_ID = process.env.PLAYWRIGHT_JUNIT_SUITE_ID;
-  process.env.FOLIO_JUNIT_SUITE_NAME = process.env.PLAYWRIGHT_JUNIT_SUITE_NAME;
-  process.env.FOLIO_JSON_OUTPUT_NAME = process.env.PLAYWRIGHT_JSON_OUTPUT_NAME;
-
   const result = await runner.run(!!opts.list, args.map(forceRegExp), opts.project || undefined);
   if (result === 'sigint')
     process.exit(130);
@@ -172,13 +158,3 @@ function overridesFromOptions(options: { [key: string]: any }): Config {
     workers: options.workers ? parseInt(options.workers, 10) : undefined,
   };
 }
-
-function addStubTestCommand(program: commander.CommanderStatic) {
-  const command = program.command('test');
-  command.description('Run tests with Playwright Test. Available in @playwright/test package.');
-  command.action(async (args, opts) => {
-    console.error('Please install @playwright/test package to use Playwright Test.');
-    console.error('  npm install -D @playwright/test');
-    process.exit(1);
-  });
-}

src/test/dispatcher.ts

@@ -0,0 +1,367 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import child_process from 'child_process';
+import path from 'path';
+import { EventEmitter } from 'events';
+import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams } from './ipc';
+import type { TestResult, Reporter, TestStatus } from './reporter';
+import { Suite, Test } from './test';
+import { Loader } from './loader';
+
+type DispatcherEntry = {
+  runPayload: RunPayload;
+  hash: string;
+  repeatEachIndex: number;
+  projectIndex: number;
+};
+
+export class Dispatcher {
+  private _workers = new Set<Worker>();
+  private _freeWorkers: Worker[] = [];
+  private _workerClaimers: (() => void)[] = [];
+
+  private _testById = new Map<string, { test: Test, result: TestResult }>();
+  private _queue: DispatcherEntry[] = [];
+  private _stopCallback = () => {};
+  readonly _loader: Loader;
+  private _suite: Suite;
+  private _reporter: Reporter;
+  private _hasWorkerErrors = false;
+  private _isStopped = false;
+  private _failureCount = 0;
+
+  constructor(loader: Loader, suite: Suite, reporter: Reporter) {
+    this._loader = loader;
+    this._reporter = reporter;
+
+    this._suite = suite;
+    for (const suite of this._suite.suites) {
+      for (const spec of suite._allSpecs()) {
+        for (const test of spec.tests)
+          this._testById.set(test._id, { test, result: test._appendTestResult() });
+      }
+    }
+
+    this._queue = this._filesSortedByWorkerHash();
+
+    // Shard tests.
+    const shard = this._loader.fullConfig().shard;
+    if (shard) {
+      let total = this._suite.totalTestCount();
+      const shardSize = Math.ceil(total / shard.total);
+      const from = shardSize * shard.current;
+      const to = shardSize * (shard.current + 1);
+      let current = 0;
+      total = 0;
+      const filteredQueue: DispatcherEntry[] = [];
+      for (const entry of this._queue) {
+        if (current >= from && current < to) {
+          filteredQueue.push(entry);
+          total += entry.runPayload.entries.length;
+        }
+        current += entry.runPayload.entries.length;
+      }
+      this._queue = filteredQueue;
+    }
+  }
+
+  _filesSortedByWorkerHash(): DispatcherEntry[] {
+    const entriesByWorkerHashAndFile = new Map<string, Map<string, DispatcherEntry>>();
+    for (const fileSuite of this._suite.suites) {
+      const file = fileSuite.file;
+      for (const spec of fileSuite._allSpecs()) {
+        for (const test of spec.tests) {
+          let entriesByFile = entriesByWorkerHashAndFile.get(test._workerHash);
+          if (!entriesByFile) {
+            entriesByFile = new Map();
+            entriesByWorkerHashAndFile.set(test._workerHash, entriesByFile);
+          }
+          let entry = entriesByFile.get(file);
+          if (!entry) {
+            entry = {
+              runPayload: {
+                entries: [],
+                file,
+              },
+              repeatEachIndex: test._repeatEachIndex,
+              projectIndex: test._projectIndex,
+              hash: test._workerHash,
+            };
+            entriesByFile.set(file, entry);
+          }
+          entry.runPayload.entries.push({
+            retry: this._testById.get(test._id)!.result.retry,
+            testId: test._id,
+          });
+        }
+      }
+    }
+
+    const result: DispatcherEntry[] = [];
+    for (const entriesByFile of entriesByWorkerHashAndFile.values()) {
+      for (const entry of entriesByFile.values())
+        result.push(entry);
+    }
+    result.sort((a, b) => a.hash < b.hash ? -1 : (a.hash === b.hash ? 0 : 1));
+    return result;
+  }
+
+  async run() {
+    // Loop in case job schedules more jobs
+    while (this._queue.length && !this._isStopped)
+      await this._dispatchQueue();
+  }
+
+  async _dispatchQueue() {
+    const jobs = [];
+    while (this._queue.length) {
+      if (this._isStopped)
+        break;
+      const entry = this._queue.shift()!;
+      const requiredHash = entry.hash;
+      let worker = await this._obtainWorker(entry);
+      while (!this._isStopped && worker.hash && worker.hash !== requiredHash) {
+        worker.stop();
+        worker = await this._obtainWorker(entry);
+      }
+      if (this._isStopped)
+        break;
+      jobs.push(this._runJob(worker, entry));
+    }
+    await Promise.all(jobs);
+  }
+
+  async _runJob(worker: Worker, entry: DispatcherEntry) {
+    worker.run(entry.runPayload);
+    let doneCallback = () => {};
+    const result = new Promise<void>(f => doneCallback = f);
+    worker.once('done', (params: DonePayload) => {
+      // We won't file remaining if:
+      // - there are no remaining
+      // - we are here not because something failed
+      // - no unrecoverable worker error
+      if (!params.remaining.length && !params.failedTestId && !params.fatalError) {
+        this._freeWorkers.push(worker);
+        this._notifyWorkerClaimer();
+        doneCallback();
+        return;
+      }
+
+      // When worker encounters error, we will stop it and create a new one.
+      worker.stop();
+
+      let remaining = params.remaining;
+      const failedTestIds = new Set<string>();
+
+      // In case of fatal error, report all remaining tests as failing with this error.
+      if (params.fatalError) {
+        for (const { testId } of remaining) {
+          const { test, result } = this._testById.get(testId)!;
+          this._reporter.onTestBegin?.(test);
+          result.error = params.fatalError;
+          this._reportTestEnd(test, result, 'failed');
+          failedTestIds.add(testId);
+        }
+        // Since we pretent that all remaining tests failed, there is nothing else to run,
+        // except for possible retries.
+        remaining = [];
+      }
+      if (params.failedTestId)
+        failedTestIds.add(params.failedTestId);
+
+      // Only retry expected failures, not passes and only if the test failed.
+      for (const testId of failedTestIds) {
+        const pair = this._testById.get(testId)!;
+        if (pair.test.expectedStatus === 'passed' && pair.test.results.length < pair.test.retries + 1) {
+          pair.result = pair.test._appendTestResult();
+          remaining.unshift({
+            retry: pair.result.retry,
+            testId: pair.test._id,
+          });
+        }
+      }
+
+      if (remaining.length)
+        this._queue.unshift({ ...entry, runPayload: { ...entry.runPayload, entries: remaining } });
+
+      // This job is over, we just scheduled another one.
+      doneCallback();
+    });
+    return result;
+  }
+
+  async _obtainWorker(entry: DispatcherEntry) {
+    const claimWorker = (): Promise<Worker> | null => {
+      // Use available worker.
+      if (this._freeWorkers.length)
+        return Promise.resolve(this._freeWorkers.pop()!);
+      // Create a new worker.
+      if (this._workers.size < this._loader.fullConfig().workers)
+        return this._createWorker(entry);
+      return null;
+    };
+
+    // Note: it is important to claim the worker synchronously,
+    // so that we won't miss a _notifyWorkerClaimer call while awaiting.
+    let worker = claimWorker();
+    if (!worker) {
+      // Wait for available or stopped worker.
+      await new Promise<void>(f => this._workerClaimers.push(f));
+      worker = claimWorker();
+    }
+    return worker!;
+  }
+
+  async _notifyWorkerClaimer() {
+    if (this._isStopped || !this._workerClaimers.length)
+      return;
+    const callback = this._workerClaimers.shift()!;
+    callback();
+  }
+
+  _createWorker(entry: DispatcherEntry) {
+    const worker = new Worker(this);
+    worker.on('testBegin', (params: TestBeginPayload) => {
+      const { test, result: testRun  } = this._testById.get(params.testId)!;
+      testRun.workerIndex = params.workerIndex;
+      this._reporter.onTestBegin(test);
+    });
+    worker.on('testEnd', (params: TestEndPayload) => {
+      const { test, result } = this._testById.get(params.testId)!;
+      result.duration = params.duration;
+      result.error = params.error;
+      test.expectedStatus = params.expectedStatus;
+      test.annotations = params.annotations;
+      test.timeout = params.timeout;
+      if (params.expectedStatus === 'skipped' && params.status === 'skipped')
+        test.skipped = true;
+      this._reportTestEnd(test, result, params.status);
+    });
+    worker.on('stdOut', (params: TestOutputPayload) => {
+      const chunk = chunkFromParams(params);
+      const pair = params.testId ? this._testById.get(params.testId) : undefined;
+      if (pair)
+        pair.result.stdout.push(chunk);
+      this._reporter.onStdOut(chunk, pair ? pair.test : undefined);
+    });
+    worker.on('stdErr', (params: TestOutputPayload) => {
+      const chunk = chunkFromParams(params);
+      const pair = params.testId ? this._testById.get(params.testId) : undefined;
+      if (pair)
+        pair.result.stderr.push(chunk);
+      this._reporter.onStdErr(chunk, pair ? pair.test : undefined);
+    });
+    worker.on('teardownError', ({error}) => {
+      this._hasWorkerErrors = true;
+      this._reporter.onError(error);
+    });
+    worker.on('exit', () => {
+      this._workers.delete(worker);
+      this._notifyWorkerClaimer();
+      if (this._stopCallback && !this._workers.size)
+        this._stopCallback();
+    });
+    this._workers.add(worker);
+    return worker.init(entry).then(() => worker);
+  }
+
+  async stop() {
+    this._isStopped = true;
+    if (this._workers.size) {
+      const result = new Promise<void>(f => this._stopCallback = f);
+      for (const worker of this._workers)
+        worker.stop();
+      await result;
+    }
+  }
+
+  private _reportTestEnd(test: Test, result: TestResult, status: TestStatus) {
+    if (this._isStopped)
+      return;
+    result.status = status;
+    if (result.status !== 'skipped' && result.status !== test.expectedStatus)
+      ++this._failureCount;
+    const maxFailures = this._loader.fullConfig().maxFailures;
+    if (!maxFailures || this._failureCount <= maxFailures)
+      this._reporter.onTestEnd(test, result);
+    if (maxFailures && this._failureCount === maxFailures)
+      this._isStopped = true;
+  }
+
+  hasWorkerErrors(): boolean {
+    return this._hasWorkerErrors;
+  }
+}
+
+let lastWorkerIndex = 0;
+
+class Worker extends EventEmitter {
+  process: child_process.ChildProcess;
+  runner: Dispatcher;
+  hash = '';
+  index: number;
+
+  constructor(runner: Dispatcher) {
+    super();
+    this.runner = runner;
+    this.index = lastWorkerIndex++;
+
+    this.process = child_process.fork(path.join(__dirname, 'worker.js'), {
+      detached: false,
+      env: {
+        FORCE_COLOR: process.stdout.isTTY ? '1' : '0',
+        DEBUG_COLORS: process.stdout.isTTY ? '1' : '0',
+        TEST_WORKER_INDEX: String(this.index),
+        ...process.env
+      },
+      // Can't pipe since piping slows down termination for some reason.
+      stdio: ['ignore', 'ignore', process.env.PW_RUNNER_DEBUG ? 'inherit' : 'ignore', 'ipc']
+    });
+    this.process.on('exit', () => this.emit('exit'));
+    this.process.on('error', e => {});  // do not yell at a send to dead process.
+    this.process.on('message', (message: any) => {
+      const { method, params } = message;
+      this.emit(method, params);
+    });
+  }
+
+  async init(entry: DispatcherEntry) {
+    this.hash = entry.hash;
+    const params: WorkerInitParams = {
+      workerIndex: this.index,
+      repeatEachIndex: entry.repeatEachIndex,
+      projectIndex: entry.projectIndex,
+      loader: this.runner._loader.serialize(),
+    };
+    this.process.send({ method: 'init', params });
+    await new Promise(f => this.process.once('message', f));  // Ready ack
+  }
+
+  run(runPayload: RunPayload) {
+    this.process.send({ method: 'run', params: runPayload });
+  }
+
+  stop() {
+    this.process.send({ method: 'stop' });
+  }
+}
+
+function chunkFromParams(params: TestOutputPayload): string | Buffer {
+  if (typeof params.text === 'string')
+    return params.text;
+  return Buffer.from(params.buffer!, 'base64');
+}

src/test/expect.ts

@@ -0,0 +1,40 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Expect } from './types';
+import expectLibrary from 'expect';
+import { currentTestInfo } from './globals';
+import { compare } from './golden';
+
+export const expect: Expect = expectLibrary;
+
+function toMatchSnapshot(received: Buffer | string, nameOrOptions: string | { name: string, threshold?: number }, optOptions: { threshold?: number } = {}) {
+  let options: { name: string, threshold?: number };
+  const testInfo = currentTestInfo();
+  if (!testInfo)
+    throw new Error(`toMatchSnapshot() must be called during the test`);
+  if (typeof nameOrOptions === 'string')
+    options = { name: nameOrOptions, ...optOptions };
+  else
+    options = { ...nameOrOptions };
+  if (!options.name)
+    throw new Error(`toMatchSnapshot() requires a "name" parameter`);
+
+  const { pass, message } = compare(received, options.name, testInfo.snapshotPath, testInfo.outputPath, testInfo.config.updateSnapshots, options);
+  return { pass, message: () => message };
+}
+
+expectLibrary.extend({ toMatchSnapshot });

src/test/fixtures.ts

@@ -0,0 +1,361 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { errorWithCallLocation, formatLocation, prependErrorMessage, wrapInPromise } from './util';
+import * as crypto from 'crypto';
+import { FixturesWithLocation, Location } from './types';
+
+type FixtureScope = 'test' | 'worker';
+type FixtureRegistration = {
+  location: Location;
+  name: string;
+  scope: FixtureScope;
+  fn: Function | any;  // Either a fixture function, or a fixture value.
+  auto: boolean;
+  deps: string[];
+  id: string;
+  super?: FixtureRegistration;
+};
+
+class Fixture {
+  runner: FixtureRunner;
+  registration: FixtureRegistration;
+  usages: Set<Fixture>;
+  value: any;
+  _teardownFenceCallback!: (value?: unknown) => void;
+  _tearDownComplete!: Promise<void>;
+  _setup = false;
+  _teardown = false;
+
+  constructor(runner: FixtureRunner, registration: FixtureRegistration) {
+    this.runner = runner;
+    this.registration = registration;
+    this.usages = new Set();
+    this.value = null;
+  }
+
+  async setup(info: any) {
+    if (typeof this.registration.fn !== 'function') {
+      this._setup = true;
+      this.value = this.registration.fn;
+      return;
+    }
+
+    const params: { [key: string]: any } = {};
+    for (const name of this.registration.deps) {
+      const registration = this.runner.pool!.resolveDependency(this.registration, name);
+      if (!registration)
+        throw errorWithCallLocation(`Unknown fixture "${name}"`);
+      const dep = await this.runner.setupFixtureForRegistration(registration, info);
+      dep.usages.add(this);
+      params[name] = dep.value;
+    }
+
+    let setupFenceFulfill = () => {};
+    let setupFenceReject = (e: Error) => {};
+    let called = false;
+    const setupFence = new Promise<void>((f, r) => { setupFenceFulfill = f; setupFenceReject = r; });
+    const teardownFence = new Promise(f => this._teardownFenceCallback = f);
+    this._tearDownComplete = wrapInPromise(this.registration.fn(params, async (value: any) => {
+      if (called)
+        throw errorWithCallLocation(`Cannot provide fixture value for the second time`);
+      called = true;
+      this.value = value;
+      setupFenceFulfill();
+      return await teardownFence;
+    }, info)).catch((e: any) => {
+      if (!this._setup)
+        setupFenceReject(e);
+      else
+        throw e;
+    });
+    await setupFence;
+    this._setup = true;
+  }
+
+  async teardown() {
+    if (this._teardown)
+      return;
+    this._teardown = true;
+    if (typeof this.registration.fn !== 'function')
+      return;
+    for (const fixture of this.usages)
+      await fixture.teardown();
+    this.usages.clear();
+    if (this._setup) {
+      this._teardownFenceCallback();
+      await this._tearDownComplete;
+    }
+    this.runner.instanceForId.delete(this.registration.id);
+  }
+}
+
+export class FixturePool {
+  readonly digest: string;
+  readonly registrations: Map<string, FixtureRegistration>;
+
+  constructor(fixturesList: FixturesWithLocation[], parentPool?: FixturePool) {
+    this.registrations = new Map(parentPool ? parentPool.registrations : []);
+
+    for (const { fixtures, location } of fixturesList) {
+      try {
+        for (const entry of Object.entries(fixtures)) {
+          const name = entry[0];
+          let value = entry[1];
+          let options: { auto: boolean, scope: FixtureScope } | undefined;
+          if (Array.isArray(value) && typeof value[1] === 'object' && ('scope' in value[1] || 'auto' in value[1])) {
+            options = {
+              auto: !!value[1].auto,
+              scope: value[1].scope || 'test'
+            };
+            value = value[0];
+          }
+          const fn = value as (Function | any);
+
+          const previous = this.registrations.get(name);
+          if (previous && options) {
+            if (previous.scope !== options.scope)
+              throw errorWithLocations(`Fixture "${name}" has already been registered as a { scope: '${previous.scope}' } fixture.`, { location, name}, previous);
+            if (previous.auto !== options.auto)
+              throw errorWithLocations(`Fixture "${name}" has already been registered as a { auto: '${previous.scope}' } fixture.`, { location, name }, previous);
+          } else if (previous) {
+            options = { auto: previous.auto, scope: previous.scope };
+          } else if (!options) {
+            options = { auto: false, scope: 'test' };
+          }
+
+          const deps = fixtureParameterNames(fn, location);
+          const registration: FixtureRegistration = { id: '', name, location, scope: options.scope, fn, auto: options.auto, deps, super: previous };
+          registrationId(registration);
+          this.registrations.set(name, registration);
+        }
+      } catch (e) {
+        prependErrorMessage(e, `Error processing fixtures at ${formatLocation(location)}:\n`);
+        throw e;
+      }
+    }
+
+    this.digest = this.validate();
+  }
+
+  private validate() {
+    const markers = new Map<FixtureRegistration, 'visiting' | 'visited'>();
+    const stack: FixtureRegistration[] = [];
+    const visit = (registration: FixtureRegistration) => {
+      markers.set(registration, 'visiting');
+      stack.push(registration);
+      for (const name of registration.deps) {
+        const dep = this.resolveDependency(registration, name);
+        if (!dep) {
+          if (name === registration.name)
+            throw errorWithLocations(`Fixture "${registration.name}" references itself, but does not have a base implementation.`, registration);
+          else
+            throw errorWithLocations(`Fixture "${registration.name}" has unknown parameter "${name}".`, registration);
+        }
+        if (registration.scope === 'worker' && dep.scope === 'test')
+          throw errorWithLocations(`Worker fixture "${registration.name}" cannot depend on a test fixture "${name}".`, registration, dep);
+        if (!markers.has(dep)) {
+          visit(dep);
+        } else if (markers.get(dep) === 'visiting') {
+          const index = stack.indexOf(dep);
+          const regs = stack.slice(index, stack.length);
+          const names = regs.map(r => `"${r.name}"`);
+          throw errorWithLocations(`Fixtures ${names.join(' -> ')} -> "${dep.name}" form a dependency cycle.`, ...regs);
+        }
+      }
+      markers.set(registration, 'visited');
+      stack.pop();
+    };
+
+    const hash = crypto.createHash('sha1');
+    const names = Array.from(this.registrations.keys()).sort();
+    for (const name of names) {
+      const registration = this.registrations.get(name)!;
+      visit(registration);
+      if (registration.scope === 'worker')
+        hash.update(registration.id + ';');
+    }
+    return hash.digest('hex');
+  }
+
+  validateFunction(fn: Function, prefix: string, allowTestFixtures: boolean, location: Location) {
+    const visit = (registration: FixtureRegistration) => {
+      for (const name of registration.deps)
+        visit(this.resolveDependency(registration, name)!);
+    };
+    for (const name of fixtureParameterNames(fn, location)) {
+      const registration = this.registrations.get(name);
+      if (!registration)
+        throw errorWithLocations(`${prefix} has unknown parameter "${name}".`, { location, name: prefix, quoted: false });
+      if (!allowTestFixtures && registration.scope === 'test')
+        throw errorWithLocations(`${prefix} cannot depend on a test fixture "${name}".`, { location, name: prefix, quoted: false }, registration);
+      visit(registration);
+    }
+  }
+
+  resolveDependency(registration: FixtureRegistration, name: string): FixtureRegistration | undefined {
+    if (name === registration.name)
+      return registration.super;
+    return this.registrations.get(name);
+  }
+}
+
+export class FixtureRunner {
+  private testScopeClean = true;
+  pool: FixturePool | undefined;
+  instanceForId = new Map<string, Fixture>();
+
+  setPool(pool: FixturePool) {
+    if (!this.testScopeClean)
+      throw new Error('Did not teardown test scope');
+    if (this.pool && pool.digest !== this.pool.digest)
+      throw new Error('Digests do not match');
+    this.pool = pool;
+  }
+
+  async teardownScope(scope: string) {
+    for (const [, fixture] of this.instanceForId) {
+      if (fixture.registration.scope === scope)
+        await fixture.teardown();
+    }
+    if (scope === 'test')
+      this.testScopeClean = true;
+  }
+
+  async resolveParametersAndRunHookOrTest(fn: Function, scope: FixtureScope, info: any) {
+    // Install all automatic fixtures.
+    for (const registration of this.pool!.registrations.values()) {
+      const shouldSkip = scope === 'worker' && registration.scope === 'test';
+      if (registration.auto && !shouldSkip)
+        await this.setupFixtureForRegistration(registration, info);
+    }
+
+    // Install used fixtures.
+    const names = fixtureParameterNames(fn, { file: '<unused>', line: 1, column: 1 });
+    const params: { [key: string]: any } = {};
+    for (const name of names) {
+      const registration = this.pool!.registrations.get(name);
+      if (!registration)
+        throw errorWithCallLocation('Unknown fixture: ' + name);
+      const fixture = await this.setupFixtureForRegistration(registration, info);
+      params[name] = fixture.value;
+    }
+
+    return fn(params, info);
+  }
+
+  async setupFixtureForRegistration(registration: FixtureRegistration, info: any): Promise<Fixture> {
+    if (registration.scope === 'test')
+      this.testScopeClean = false;
+
+    let fixture = this.instanceForId.get(registration.id);
+    if (fixture)
+      return fixture;
+
+    fixture = new Fixture(this, registration);
+    this.instanceForId.set(registration.id, fixture);
+    await fixture.setup(info);
+    return fixture;
+  }
+}
+
+export function inheritFixtureParameterNames(from: Function, to: Function, location: Location) {
+  if (!(to as any)[signatureSymbol])
+    (to as any)[signatureSymbol] = innerFixtureParameterNames(from, location);
+}
+
+const signatureSymbol = Symbol('signature');
+
+function fixtureParameterNames(fn: Function | any, location: Location): string[] {
+  if (typeof fn !== 'function')
+    return [];
+  if (!fn[signatureSymbol])
+    fn[signatureSymbol] = innerFixtureParameterNames(fn, location);
+  return fn[signatureSymbol];
+}
+
+function innerFixtureParameterNames(fn: Function, location: Location): string[] {
+  const text = fn.toString();
+  const match = text.match(/(?:async)?(?:\s+function)?[^(]*\(([^)]*)/);
+  if (!match)
+    return [];
+  const trimmedParams = match[1].trim();
+  if (!trimmedParams)
+    return [];
+  const [firstParam] = splitByComma(trimmedParams);
+  if (firstParam[0] !== '{' || firstParam[firstParam.length - 1] !== '}')
+    throw errorWithLocations('First argument must use the object destructuring pattern: '  + firstParam, { location });
+  const props = splitByComma(firstParam.substring(1, firstParam.length - 1)).map(prop => {
+    const colon = prop.indexOf(':');
+    return colon === -1 ? prop : prop.substring(0, colon).trim();
+  });
+  return props;
+}
+
+function splitByComma(s: string) {
+  const result: string[] = [];
+  const stack: string[] = [];
+  let start = 0;
+  for (let i = 0; i < s.length; i++) {
+    if (s[i] === '{' || s[i] === '[') {
+      stack.push(s[i] === '{' ? '}' : ']');
+    } else if (s[i] === stack[stack.length - 1]) {
+      stack.pop();
+    } else if (!stack.length && s[i] === ',') {
+      const token = s.substring(start, i).trim();
+      if (token)
+        result.push(token);
+      start = i + 1;
+    }
+  }
+  const lastToken = s.substring(start).trim();
+  if (lastToken)
+    result.push(lastToken);
+  return result;
+}
+
+// name + superId, fn -> id
+const registrationIdMap = new Map<string, Map<Function | any, string>>();
+let lastId = 0;
+
+function registrationId(registration: FixtureRegistration): string {
+  if (registration.id)
+    return registration.id;
+  const key = registration.name + '@@@' + (registration.super ?  registrationId(registration.super) : '');
+  let map = registrationIdMap.get(key);
+  if (!map) {
+    map = new Map();
+    registrationIdMap.set(key, map);
+  }
+  if (!map.has(registration.fn))
+    map.set(registration.fn, String(lastId++));
+  registration.id = map.get(registration.fn)!;
+  return registration.id;
+}
+
+function errorWithLocations(message: string, ...defined: { location: Location, name?: string, quoted?: boolean }[]): Error {
+  for (const { name, location, quoted } of defined) {
+    let prefix = '';
+    if (name && quoted === false)
+      prefix = name + ' ';
+    else if (name)
+      prefix = `"${name}" `;
+    message += `\n  ${prefix}defined at ${formatLocation(location)}`;
+  }
+  const error = new Error(message);
+  error.stack = 'Error: ' + message + '\n';
+  return error;
+}

src/test/globals.ts

@@ -0,0 +1,34 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { TestInfo } from './types';
+import { Suite } from './test';
+
+let currentTestInfoValue: TestInfo | null = null;
+export function setCurrentTestInfo(testInfo: TestInfo | null) {
+  currentTestInfoValue = testInfo;
+}
+export function currentTestInfo(): TestInfo | null {
+  return currentTestInfoValue;
+}
+
+let currentFileSuite: Suite | undefined;
+export function setCurrentlyLoadingFileSuite(suite: Suite | undefined) {
+  currentFileSuite = suite;
+}
+export function currentlyLoadingFileSuite() {
+  return currentFileSuite;
+}

src/test/golden.ts

@@ -0,0 +1,179 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import colors from 'colors/safe';
+import fs from 'fs';
+import path from 'path';
+import jpeg from 'jpeg-js';
+import pixelmatch from 'pixelmatch';
+import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch';
+import { UpdateSnapshots } from './types';
+
+// Note: we require the pngjs version of pixelmatch to avoid version mismatches.
+const { PNG } = require(require.resolve('pngjs', { paths: [require.resolve('pixelmatch')] }));
+
+const extensionToMimeType: { [key: string]: string } = {
+  'dat': 'application/octet-string',
+  'jpeg': 'image/jpeg',
+  'jpg': 'image/jpeg',
+  'png': 'image/png',
+  'txt': 'text/plain',
+};
+
+const GoldenComparators: { [key: string]: any } = {
+  'application/octet-string': compareBuffers,
+  'image/png': compareImages,
+  'image/jpeg': compareImages,
+  'text/plain': compareText,
+};
+
+function compareBuffers(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string): { diff?: object; errorMessage?: string; } | null {
+  if (!actualBuffer || !(actualBuffer instanceof Buffer))
+    return { errorMessage: 'Actual result should be Buffer.' };
+  if (Buffer.compare(actualBuffer, expectedBuffer))
+    return { errorMessage: 'Buffers differ' };
+  return null;
+}
+
+function compareImages(actualBuffer: Buffer | string, expectedBuffer: Buffer, mimeType: string, options = {}): { diff?: object; errorMessage?: string; } | null {
+  if (!actualBuffer || !(actualBuffer instanceof Buffer))
+    return { errorMessage: 'Actual result should be Buffer.' };
+
+  const actual = mimeType === 'image/png' ? PNG.sync.read(actualBuffer) : jpeg.decode(actualBuffer);
+  const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer);
+  if (expected.width !== actual.width || expected.height !== actual.height) {
+    return {
+      errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
+    };
+  }
+  const diff = new PNG({width: expected.width, height: expected.height});
+  const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { threshold: 0.2, ...options });
+  return count > 0 ? { diff: PNG.sync.write(diff) } : null;
+}
+
+function compareText(actual: Buffer | string, expectedBuffer: Buffer): { diff?: object; errorMessage?: string; diffExtension?: string; } | null {
+  if (typeof actual !== 'string')
+    return { errorMessage: 'Actual result should be string' };
+  const expected = expectedBuffer.toString('utf-8');
+  if (expected === actual)
+    return null;
+  const dmp = new diff_match_patch();
+  const d = dmp.diff_main(expected, actual);
+  dmp.diff_cleanupSemantic(d);
+  return {
+    errorMessage: diff_prettyTerminal(d)
+  };
+}
+
+export function compare(actual: Buffer | string, name: string, snapshotPath: (name: string) => string, outputPath: (name: string) => string, updateSnapshots: UpdateSnapshots, options?: { threshold?: number }): { pass: boolean; message?: string; } {
+  const snapshotFile = snapshotPath(name);
+  if (!fs.existsSync(snapshotFile)) {
+    const writingActual = updateSnapshots === 'all' || updateSnapshots === 'missing';
+    if (writingActual) {
+      fs.mkdirSync(path.dirname(snapshotFile), { recursive: true });
+      fs.writeFileSync(snapshotFile, actual);
+    }
+    const message = snapshotFile + ' is missing in snapshots' + (writingActual ? ', writing actual.' : '.');
+    if (updateSnapshots === 'all') {
+      console.log(message);
+      return { pass: true, message };
+    }
+    return { pass: false, message };
+  }
+  const expected = fs.readFileSync(snapshotFile);
+  const extension = path.extname(snapshotFile).substring(1);
+  const mimeType = extensionToMimeType[extension] || 'application/octet-string';
+  const comparator = GoldenComparators[mimeType];
+  if (!comparator) {
+    return {
+      pass: false,
+      message: 'Failed to find comparator with type ' + mimeType + ': '  + snapshotFile,
+    };
+  }
+
+  const result = comparator(actual, expected, mimeType, options);
+  if (!result)
+    return { pass: true };
+
+  if (updateSnapshots === 'all') {
+    fs.mkdirSync(path.dirname(snapshotFile), { recursive: true });
+    fs.writeFileSync(snapshotFile, actual);
+    console.log(snapshotFile + ' does not match, writing actual.');
+    return {
+      pass: true,
+      message: snapshotFile + ' running with --update-snapshots, writing actual.'
+    };
+  }
+  const outputFile = outputPath(name);
+  const expectedPath = addSuffix(outputFile, '-expected');
+  const actualPath = addSuffix(outputFile, '-actual');
+  const diffPath = addSuffix(outputFile, '-diff');
+  fs.writeFileSync(expectedPath, expected);
+  fs.writeFileSync(actualPath, actual);
+  if (result.diff)
+    fs.writeFileSync(diffPath, result.diff);
+
+  const output = [
+    colors.red(`Snapshot comparison failed:`),
+  ];
+  if (result.errorMessage) {
+    output.push('');
+    output.push(indent(result.errorMessage, '  '));
+  }
+  output.push('');
+  output.push(`Expected: ${colors.yellow(expectedPath)}`);
+  output.push(`Received: ${colors.yellow(actualPath)}`);
+  if (result.diff)
+    output.push(`    Diff: ${colors.yellow(diffPath)}`);
+
+  return {
+    pass: false,
+    message: output.join('\n'),
+  };
+}
+
+function indent(lines: string, tab: string) {
+  return lines.replace(/^(?=.+$)/gm, tab);
+}
+
+function addSuffix(filePath: string, suffix: string, customExtension?: string): string {
+  const dirname = path.dirname(filePath);
+  const ext = path.extname(filePath);
+  const name = path.basename(filePath, ext);
+  return path.join(dirname, name + suffix + (customExtension || ext));
+}
+
+function diff_prettyTerminal(diffs: diff_match_patch.Diff[]) {
+  const html = [];
+  for (let x = 0; x < diffs.length; x++) {
+    const op = diffs[x][0];    // Operation (insert, delete, equal)
+    const data = diffs[x][1];  // Text of change.
+    const text = data;
+    switch (op) {
+      case DIFF_INSERT:
+        html[x] = colors.green(text);
+        break;
+      case DIFF_DELETE:
+        html[x] = colors.strikethrough(colors.red(text));
+        break;
+      case DIFF_EQUAL:
+        html[x] = text;
+        break;
+    }
+  }
+  return html.join('');
+}

src/test/index.ts

@@ -15,12 +15,12 @@
  */
 
 import * as fs from 'fs';
-import * as folio from 'folio';
+import { test as base } from './internal';
 import type { LaunchOptions, BrowserContextOptions, Page } from '../../types/types';
 import type { PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions } from '../../types/test';
 
-export * from 'folio';
+export * from './internal';
-export const test = folio.test.extend<PlaywrightTestArgs & PlaywrightTestOptions, PlaywrightWorkerArgs & PlaywrightWorkerOptions>({
+export const test = base.extend<PlaywrightTestArgs & PlaywrightTestOptions, PlaywrightWorkerArgs & PlaywrightWorkerOptions>({
   defaultBrowserType: [ 'chromium', { scope: 'worker' } ],
   browserName: [ ({ defaultBrowserType }, use) => use(defaultBrowserType), { scope: 'worker' } ],
   playwright: [ require('../inprocess'), { scope: 'worker' } ],
@@ -149,5 +149,3 @@ export const test = folio.test.extend<PlaywrightTestArgs & PlaywrightTestOptions
   },
 });
 export default test;
-
-export const __baseTest = folio.test;

src/test/internal.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2019 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { TestType } from './types';
+import { rootTestType } from './testType';
+
+export type { Project, Config, TestStatus, TestInfo, WorkerInfo, TestType, Fixtures, TestFixture, WorkerFixture } from './types';
+
+export { expect } from './expect';
+export const test: TestType<{}, {}> = rootTestType.test;
+export default test;

src/test/ipc.ts

@@ -0,0 +1,67 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { TestError } from './reporter';
+import type { Config, TestStatus } from './types';
+
+export type SerializedLoaderData = {
+  defaultConfig: Config;
+  overrides: Config;
+  configFile: { file: string } | { rootDir: string };
+};
+export type WorkerInitParams = {
+  workerIndex: number;
+  repeatEachIndex: number;
+  projectIndex: number;
+  loader: SerializedLoaderData;
+};
+
+export type TestBeginPayload = {
+  testId: string;
+  workerIndex: number,
+};
+
+export type TestEndPayload = {
+  testId: string;
+  duration: number;
+  status: TestStatus;
+  error?: TestError;
+  expectedStatus: TestStatus;
+  annotations: { type: string, description?: string }[];
+  timeout: number;
+};
+
+export type TestEntry = {
+  testId: string;
+  retry: number;
+};
+
+export type RunPayload = {
+  file: string;
+  entries: TestEntry[];
+};
+
+export type DonePayload = {
+  failedTestId?: string;
+  fatalError?: any;
+  remaining: TestEntry[];
+};
+
+export type TestOutputPayload = {
+  testId?: string;
+  text?: string;
+  buffer?: string;
+};

src/test/loader.ts

@@ -0,0 +1,397 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { installTransform } from './transform';
+import type { FullConfig, Config, FullProject, Project, ReporterDescription, PreserveOutput } from './types';
+import { errorWithCallLocation, isRegExp, mergeObjects, prependErrorMessage } from './util';
+import { setCurrentlyLoadingFileSuite } from './globals';
+import { Suite } from './test';
+import { SerializedLoaderData } from './ipc';
+import * as path from 'path';
+import { ProjectImpl } from './project';
+import { Reporter } from './reporter';
+
+export class Loader {
+  private _defaultConfig: Config;
+  private _configOverrides: Config;
+  private _fullConfig: FullConfig;
+  private _config: Config = {};
+  private _configFile: string | undefined;
+  private _projects: ProjectImpl[] = [];
+  private _fileSuites = new Map<string, Suite>();
+
+  constructor(defaultConfig: Config, configOverrides: Config) {
+    this._defaultConfig = defaultConfig;
+    this._configOverrides = configOverrides;
+    this._fullConfig = baseFullConfig;
+  }
+
+  static deserialize(data: SerializedLoaderData): Loader {
+    const loader = new Loader(data.defaultConfig, data.overrides);
+    if ('file' in data.configFile)
+      loader.loadConfigFile(data.configFile.file);
+    else
+      loader.loadEmptyConfig(data.configFile.rootDir);
+    return loader;
+  }
+
+  loadConfigFile(file: string): Config {
+    if (this._configFile)
+      throw new Error('Cannot load two config files');
+    const revertBabelRequire = installTransform();
+    try {
+      let config = require(file);
+      if (config && typeof config === 'object' && ('default' in config))
+        config = config['default'];
+      this._config = config;
+      const rawConfig = { ...config };
+      this._processConfigObject(path.dirname(file));
+      this._configFile = file;
+      return rawConfig;
+    } catch (e) {
+      prependErrorMessage(e, `Error while reading ${file}:\n`);
+      throw e;
+    } finally {
+      revertBabelRequire();
+    }
+  }
+
+  loadEmptyConfig(rootDir: string) {
+    this._config = {};
+    this._processConfigObject(rootDir);
+  }
+
+  private _processConfigObject(rootDir: string) {
+    validateConfig(this._config);
+
+    const configUse = mergeObjects(this._defaultConfig.use, this._config.use);
+    this._config = mergeObjects(mergeObjects(this._defaultConfig, this._config), { use: configUse });
+
+    if (('testDir' in this._config) && this._config.testDir !== undefined && !path.isAbsolute(this._config.testDir))
+      this._config.testDir = path.resolve(rootDir, this._config.testDir);
+    const projects: Project[] = ('projects' in this._config) && this._config.projects !== undefined ? this._config.projects : [this._config];
+
+    this._fullConfig.rootDir = this._config.testDir || rootDir;
+    this._fullConfig.forbidOnly = takeFirst(this._configOverrides.forbidOnly, this._config.forbidOnly, baseFullConfig.forbidOnly);
+    this._fullConfig.globalSetup = takeFirst(this._configOverrides.globalSetup, this._config.globalSetup, baseFullConfig.globalSetup);
+    this._fullConfig.globalTeardown = takeFirst(this._configOverrides.globalTeardown, this._config.globalTeardown, baseFullConfig.globalTeardown);
+    this._fullConfig.globalTimeout = takeFirst(this._configOverrides.globalTimeout, this._configOverrides.globalTimeout, this._config.globalTimeout, baseFullConfig.globalTimeout);
+    this._fullConfig.grep = takeFirst(this._configOverrides.grep, this._config.grep, baseFullConfig.grep);
+    this._fullConfig.maxFailures = takeFirst(this._configOverrides.maxFailures, this._config.maxFailures, baseFullConfig.maxFailures);
+    this._fullConfig.preserveOutput = takeFirst<PreserveOutput>(this._configOverrides.preserveOutput, this._config.preserveOutput, baseFullConfig.preserveOutput);
+    this._fullConfig.reporter = takeFirst(toReporters(this._configOverrides.reporter), toReporters(this._config.reporter), baseFullConfig.reporter);
+    this._fullConfig.quiet = takeFirst(this._configOverrides.quiet, this._config.quiet, baseFullConfig.quiet);
+    this._fullConfig.shard = takeFirst(this._configOverrides.shard, this._config.shard, baseFullConfig.shard);
+    this._fullConfig.updateSnapshots = takeFirst(this._configOverrides.updateSnapshots, this._config.updateSnapshots, baseFullConfig.updateSnapshots);
+    this._fullConfig.workers = takeFirst(this._configOverrides.workers, this._config.workers, baseFullConfig.workers);
+
+    for (const project of projects)
+      this._addProject(project, this._fullConfig.rootDir);
+    this._fullConfig.projects = this._projects.map(p => p.config);
+  }
+
+  loadTestFile(file: string) {
+    if (this._fileSuites.has(file))
+      return this._fileSuites.get(file)!;
+    const revertBabelRequire = installTransform();
+    try {
+      const suite = new Suite('');
+      suite.file = file;
+      setCurrentlyLoadingFileSuite(suite);
+      require(file);
+      this._fileSuites.set(file, suite);
+      return suite;
+    } catch (e) {
+      prependErrorMessage(e, `Error while reading ${file}:\n`);
+      throw e;
+    } finally {
+      revertBabelRequire();
+      setCurrentlyLoadingFileSuite(undefined);
+    }
+  }
+
+  loadGlobalHook(file: string, name: string): (config: FullConfig) => any {
+    const revertBabelRequire = installTransform();
+    try {
+      let hook = require(file);
+      if (hook && typeof hook === 'object' && ('default' in hook))
+        hook = hook['default'];
+      if (typeof hook !== 'function')
+        throw errorWithCallLocation(`${name} file must export a single function.`);
+      return hook;
+    } catch (e) {
+      prependErrorMessage(e, `Error while reading ${file}:\n`);
+      throw e;
+    } finally {
+      revertBabelRequire();
+    }
+  }
+
+  loadReporter(file: string): new (arg?: any) => Reporter {
+    const revertBabelRequire = installTransform();
+    try {
+      let func = require(path.resolve(this._fullConfig.rootDir, file));
+      if (func && typeof func === 'object' && ('default' in func))
+        func = func['default'];
+      if (typeof func !== 'function')
+        throw errorWithCallLocation(`Reporter file "${file}" must export a single class.`);
+      return func;
+    } catch (e) {
+      prependErrorMessage(e, `Error while reading ${file}:\n`);
+      throw e;
+    } finally {
+      revertBabelRequire();
+    }
+  }
+
+  fullConfig(): FullConfig {
+    return this._fullConfig;
+  }
+
+  projects() {
+    return this._projects;
+  }
+
+  fileSuites() {
+    return this._fileSuites;
+  }
+
+  serialize(): SerializedLoaderData {
+    return {
+      defaultConfig: this._defaultConfig,
+      configFile: this._configFile ? { file: this._configFile } : { rootDir: this._fullConfig.rootDir },
+      overrides: this._configOverrides,
+    };
+  }
+
+  private _addProject(projectConfig: Project, rootDir: string) {
+    let testDir = takeFirst(projectConfig.testDir, rootDir);
+    if (!path.isAbsolute(testDir))
+      testDir = path.resolve(rootDir, testDir);
+
+    const fullProject: FullProject = {
+      define: takeFirst(this._configOverrides.define, projectConfig.define, this._config.define, []),
+      outputDir: takeFirst(this._configOverrides.outputDir, projectConfig.outputDir, this._config.outputDir, path.resolve(process.cwd(), 'test-results')),
+      repeatEach: takeFirst(this._configOverrides.repeatEach, projectConfig.repeatEach, this._config.repeatEach, 1),
+      retries: takeFirst(this._configOverrides.retries, projectConfig.retries, this._config.retries, 0),
+      metadata: takeFirst(this._configOverrides.metadata, projectConfig.metadata, this._config.metadata, undefined),
+      name: takeFirst(this._configOverrides.name, projectConfig.name, this._config.name, ''),
+      testDir,
+      testIgnore: takeFirst(this._configOverrides.testIgnore, projectConfig.testIgnore, this._config.testIgnore, []),
+      testMatch: takeFirst(this._configOverrides.testMatch, projectConfig.testMatch, this._config.testMatch, '**/?(*.)+(spec|test).[jt]s'),
+      timeout: takeFirst(this._configOverrides.timeout, projectConfig.timeout, this._config.timeout, 10000),
+      use: mergeObjects(mergeObjects(this._config.use, projectConfig.use), this._configOverrides.use),
+    };
+    this._projects.push(new ProjectImpl(fullProject, this._projects.length));
+  }
+}
+
+function takeFirst<T>(...args: (T | undefined)[]): T {
+  for (const arg of args) {
+    if (arg !== undefined)
+      return arg;
+  }
+  return undefined as any as T;
+}
+
+function toReporters(reporters: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'null' | ReporterDescription[] | undefined): ReporterDescription[] | undefined {
+  if (!reporters)
+    return;
+  if (typeof reporters === 'string')
+    return [ [reporters] ];
+  return reporters;
+}
+
+function validateConfig(config: Config) {
+  if (typeof config !== 'object' || !config)
+    throw new Error(`Configuration file must export a single object`);
+
+  validateProject(config, 'config');
+
+  if ('forbidOnly' in config && config.forbidOnly !== undefined) {
+    if (typeof config.forbidOnly !== 'boolean')
+      throw new Error(`config.forbidOnly must be a boolean`);
+  }
+
+  if ('globalSetup' in config && config.globalSetup !== undefined) {
+    if (typeof config.globalSetup !== 'string')
+      throw new Error(`config.globalSetup must be a string`);
+  }
+
+  if ('globalTeardown' in config && config.globalTeardown !== undefined) {
+    if (typeof config.globalTeardown !== 'string')
+      throw new Error(`config.globalTeardown must be a string`);
+  }
+
+  if ('globalTimeout' in config && config.globalTimeout !== undefined) {
+    if (typeof config.globalTimeout !== 'number' || config.globalTimeout < 0)
+      throw new Error(`config.globalTimeout must be a non-negative number`);
+  }
+
+  if ('grep' in config && config.grep !== undefined) {
+    if (Array.isArray(config.grep)) {
+      config.grep.forEach((item, index) => {
+        if (!isRegExp(item))
+          throw new Error(`config.grep[${index}] must be a RegExp`);
+      });
+    } else if (!isRegExp(config.grep)) {
+      throw new Error(`config.grep must be a RegExp`);
+    }
+  }
+
+  if ('maxFailures' in config && config.maxFailures !== undefined) {
+    if (typeof config.maxFailures !== 'number' || config.maxFailures < 0)
+      throw new Error(`config.maxFailures must be a non-negative number`);
+  }
+
+  if ('preserveOutput' in config && config.preserveOutput !== undefined) {
+    if (typeof config.preserveOutput !== 'string' || !['always', 'never', 'failures-only'].includes(config.preserveOutput))
+      throw new Error(`config.preserveOutput must be one of "always", "never" or "failures-only"`);
+  }
+
+  if ('projects' in config && config.projects !== undefined) {
+    if (!Array.isArray(config.projects))
+      throw new Error(`config.projects must be an array`);
+    config.projects.forEach((project, index) => {
+      validateProject(project, `config.projects[${index}]`);
+    });
+  }
+
+  if ('quiet' in config && config.quiet !== undefined) {
+    if (typeof config.quiet !== 'boolean')
+      throw new Error(`config.quiet must be a boolean`);
+  }
+
+  if ('reporter' in config && config.reporter !== undefined) {
+    if (Array.isArray(config.reporter)) {
+      config.reporter.forEach((item, index) => {
+        if (!Array.isArray(item) || item.length <= 0 || item.length > 2 || typeof item[0] !== 'string')
+          throw new Error(`config.reporter[${index}] must be a tuple [name, optionalArgument]`);
+      });
+    } else {
+      const builtinReporters = ['dot', 'line', 'list', 'junit', 'json', 'null'];
+      if (typeof config.reporter !== 'string' || !builtinReporters.includes(config.reporter))
+        throw new Error(`config.reporter must be one of ${builtinReporters.map(name => `"${name}"`).join(', ')}`);
+    }
+  }
+
+  if ('shard' in config && config.shard !== undefined && config.shard !== null) {
+    if (!config.shard || typeof config.shard !== 'object')
+      throw new Error(`config.shard must be an object`);
+    if (!('total' in config.shard) || typeof config.shard.total !== 'number' || config.shard.total < 1)
+      throw new Error(`config.shard.total must be a positive number`);
+    if (!('current' in config.shard) || typeof config.shard.current !== 'number' || config.shard.current < 1 || config.shard.current > config.shard.total)
+      throw new Error(`config.shard.current must be a positive number, not greater than config.shard.total`);
+  }
+
+  if ('updateSnapshots' in config && config.updateSnapshots !== undefined) {
+    if (typeof config.updateSnapshots !== 'string' || !['all', 'none', 'missing'].includes(config.updateSnapshots))
+      throw new Error(`config.updateSnapshots must be one of "all", "none" or "missing"`);
+  }
+
+  if ('workers' in config && config.workers !== undefined) {
+    if (typeof config.workers !== 'number' || config.workers <= 0)
+      throw new Error(`config.workers must be a positive number`);
+  }
+}
+
+function validateProject(project: Project, title: string) {
+  if (typeof project !== 'object' || !project)
+    throw new Error(`${title} must be an object`);
+
+  if ('define' in project && project.define !== undefined) {
+    if (Array.isArray(project.define)) {
+      project.define.forEach((item, index) => {
+        validateDefine(item, `${title}.define[${index}]`);
+      });
+    } else {
+      validateDefine(project.define, `${title}.define`);
+    }
+  }
+
+  if ('name' in project && project.name !== undefined) {
+    if (typeof project.name !== 'string')
+      throw new Error(`${title}.name must be a string`);
+  }
+
+  if ('outputDir' in project && project.outputDir !== undefined) {
+    if (typeof project.outputDir !== 'string')
+      throw new Error(`${title}.outputDir must be a string`);
+    if (!path.isAbsolute(project.outputDir))
+      throw new Error(`${title}.outputDir must be an absolute path`);
+  }
+
+  if ('repeatEach' in project && project.repeatEach !== undefined) {
+    if (typeof project.repeatEach !== 'number' || project.repeatEach < 0)
+      throw new Error(`${title}.repeatEach must be a non-negative number`);
+  }
+
+  if ('retries' in project && project.retries !== undefined) {
+    if (typeof project.retries !== 'number' || project.retries < 0)
+      throw new Error(`${title}.retries must be a non-negative number`);
+  }
+
+  if ('testDir' in project && project.testDir !== undefined) {
+    if (typeof project.testDir !== 'string')
+      throw new Error(`${title}.testDir must be a string`);
+  }
+
+  for (const prop of ['testIgnore', 'testMatch'] as const) {
+    if (prop in project && project[prop] !== undefined) {
+      const value = project[prop];
+      if (Array.isArray(value)) {
+        value.forEach((item, index) => {
+          if (typeof item !== 'string' && !isRegExp(item))
+            throw new Error(`${title}.${prop}[${index}] must be a string or a RegExp`);
+        });
+      } else if (typeof value !== 'string' && !isRegExp(value)) {
+        throw new Error(`${title}.${prop} must be a string or a RegExp`);
+      }
+    }
+  }
+
+  if ('timeout' in project && project.timeout !== undefined) {
+    if (typeof project.timeout !== 'number' || project.timeout < 0)
+      throw new Error(`${title}.timeout must be a non-negative number`);
+  }
+
+  if ('use' in project && project.use !== undefined) {
+    if (!project.use || typeof project.use !== 'object')
+      throw new Error(`${title}.use must be an object`);
+  }
+}
+
+function validateDefine(define: any, title: string) {
+  if (!define || typeof define !== 'object' || !define.test || !define.fixtures)
+    throw new Error(`${title} must be an object with "test" and "fixtures" properties`);
+}
+
+const baseFullConfig: FullConfig = {
+  forbidOnly: false,
+  globalSetup: null,
+  globalTeardown: null,
+  globalTimeout: 0,
+  grep: /.*/,
+  maxFailures: 0,
+  preserveOutput: 'always',
+  projects: [],
+  reporter: [ ['list'] ],
+  rootDir: path.resolve(process.cwd()),
+  quiet: false,
+  shard: null,
+  updateSnapshots: 'missing',
+  workers: 1,
+};

src/test/project.ts

@@ -0,0 +1,109 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { TestType, FullProject, Fixtures, FixturesWithLocation } from './types';
+import { Spec, Test } from './test';
+import { FixturePool } from './fixtures';
+import { DeclaredFixtures, TestTypeImpl } from './testType';
+
+export class ProjectImpl {
+  config: FullProject;
+  private index: number;
+  private defines = new Map<TestType<any, any>, Fixtures>();
+  private testTypePools = new Map<TestTypeImpl, FixturePool>();
+  private specPools = new Map<Spec, FixturePool>();
+
+  constructor(project: FullProject, index: number) {
+    this.config = project;
+    this.index = index;
+    this.defines = new Map();
+    for (const { test, fixtures } of Array.isArray(project.define) ? project.define : [project.define])
+      this.defines.set(test, fixtures);
+  }
+
+  private buildTestTypePool(testType: TestTypeImpl): FixturePool {
+    if (!this.testTypePools.has(testType)) {
+      const fixtures = this.resolveFixtures(testType);
+      const overrides: Fixtures = this.config.use;
+      const overridesWithLocation = {
+        fixtures: overrides,
+        location: {
+          file: `<configuration file>`,
+          line: 1,
+          column: 1,
+        }
+      };
+      const pool = new FixturePool([...fixtures, overridesWithLocation]);
+      this.testTypePools.set(testType, pool);
+    }
+    return this.testTypePools.get(testType)!;
+  }
+
+  buildPool(spec: Spec): FixturePool {
+    if (!this.specPools.has(spec)) {
+      let pool = this.buildTestTypePool(spec._testType);
+      const overrides: Fixtures = spec.parent!._buildFixtureOverrides();
+      if (Object.entries(overrides).length) {
+        const overridesWithLocation = {
+          fixtures: overrides,
+          location: {
+            file: spec.file,
+            line: 1,  // TODO: capture location
+            column: 1,  // TODO: capture location
+          }
+        };
+        pool = new FixturePool([overridesWithLocation], pool);
+      }
+      this.specPools.set(spec, pool);
+
+      pool.validateFunction(spec.fn, 'Test', true, spec);
+      for (let parent = spec.parent; parent; parent = parent.parent) {
+        for (const hook of parent._hooks)
+          pool.validateFunction(hook.fn, hook.type + ' hook', hook.type === 'beforeEach' || hook.type === 'afterEach', hook.location);
+      }
+    }
+    return this.specPools.get(spec)!;
+  }
+
+  generateTests(spec: Spec, repeatEachIndex?: number) {
+    const digest = this.buildPool(spec).digest;
+    const min = repeatEachIndex === undefined ? 0 : repeatEachIndex;
+    const max = repeatEachIndex === undefined ? this.config.repeatEach - 1 : repeatEachIndex;
+    const tests: Test[] = [];
+    for (let i = min; i <= max; i++) {
+      const test = new Test(spec);
+      test.projectName = this.config.name;
+      test.retries = this.config.retries;
+      test._repeatEachIndex = i;
+      test._projectIndex = this.index;
+      test._workerHash = `run${this.index}-${digest}-repeat${i}`;
+      test._id = `${spec._ordinalInFile}@${spec.file}#run${this.index}-repeat${i}`;
+      spec.tests.push(test);
+      tests.push(test);
+    }
+    return tests;
+  }
+
+  private resolveFixtures(testType: TestTypeImpl): FixturesWithLocation[] {
+    return testType.fixtures.map(f => {
+      if (f instanceof DeclaredFixtures) {
+        const fixtures = this.defines.get(f.testType.test) || {};
+        return { fixtures, location: f.location };
+      }
+      return f;
+    });
+  }
+}

src/test/reporter.ts

@@ -0,0 +1,77 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { FullConfig, TestStatus } from './types';
+export type { FullConfig, TestStatus } from './types';
+
+export interface Suite {
+  title: string;
+  file: string;
+  line: number;
+  column: number;
+  suites: Suite[];
+  specs: Spec[];
+  findTest(fn: (test: Test) => boolean | void): boolean;
+  findSpec(fn: (spec: Spec) => boolean | void): boolean;
+  totalTestCount(): number;
+}
+export interface Spec {
+  suite: Suite;
+  title: string;
+  file: string;
+  line: number;
+  column: number;
+  tests: Test[];
+  fullTitle(): string;
+  ok(): boolean;
+}
+export interface Test {
+  spec: Spec;
+  results: TestResult[];
+  skipped: boolean;
+  expectedStatus: TestStatus;
+  timeout: number;
+  annotations: { type: string, description?: string }[];
+  projectName: string;
+  retries: number;
+  fullTitle(): string;
+  status(): 'skipped' | 'expected' | 'unexpected' | 'flaky';
+  ok(): boolean;
+}
+export interface TestResult {
+  retry: number;
+  workerIndex: number,
+  duration: number;
+  status?: TestStatus;
+  error?: TestError;
+  stdout: (string | Buffer)[];
+  stderr: (string | Buffer)[];
+}
+export interface TestError {
+  message?: string;
+  stack?: string;
+  value?: string;
+}
+export interface Reporter {
+  onBegin(config: FullConfig, suite: Suite): void;
+  onTestBegin(test: Test): void;
+  onStdOut(chunk: string | Buffer, test?: Test): void;
+  onStdErr(chunk: string | Buffer, test?: Test): void;
+  onTestEnd(test: Test, result: TestResult): void;
+  onTimeout(timeout: number): void;
+  onError(error: TestError): void;
+  onEnd(): void;
+}

src/test/reporters/base.ts

@@ -0,0 +1,245 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { codeFrameColumns } from '@babel/code-frame';
+import colors from 'colors/safe';
+import fs from 'fs';
+// @ts-ignore
+import milliseconds from 'ms';
+import path from 'path';
+import StackUtils from 'stack-utils';
+import { FullConfig, TestStatus, Test, Suite, TestResult, TestError, Reporter } from '../reporter';
+
+const stackUtils = new StackUtils();
+
+export class BaseReporter implements Reporter  {
+  duration = 0;
+  config!: FullConfig;
+  suite!: Suite;
+  timeout: number = 0;
+  fileDurations = new Map<string, number>();
+  monotonicStartTime: number = 0;
+
+  constructor() {
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
+    this.monotonicStartTime = monotonicTime();
+    this.config = config;
+    this.suite = suite;
+  }
+
+  onTestBegin(test: Test) {
+  }
+
+  onStdOut(chunk: string | Buffer) {
+    if (!this.config.quiet)
+      process.stdout.write(chunk);
+  }
+
+  onStdErr(chunk: string | Buffer) {
+    if (!this.config.quiet)
+      process.stderr.write(chunk);
+  }
+
+  onTestEnd(test: Test, result: TestResult) {
+    const spec = test.spec;
+    let duration = this.fileDurations.get(spec.file) || 0;
+    duration += result.duration;
+    this.fileDurations.set(spec.file, duration);
+  }
+
+  onError(error: TestError) {
+    console.log(formatError(error));
+  }
+
+  onTimeout(timeout: number) {
+    this.timeout = timeout;
+  }
+
+  onEnd() {
+    this.duration = monotonicTime() - this.monotonicStartTime;
+  }
+
+  private _printSlowTests() {
+    const fileDurations = [...this.fileDurations.entries()];
+    fileDurations.sort((a, b) => b[1] - a[1]);
+    for (let i = 0; i < 10 && i < fileDurations.length; ++i) {
+      const baseName = path.basename(fileDurations[i][0]);
+      const duration = fileDurations[i][1];
+      if (duration < 15000)
+        break;
+      console.log(colors.yellow('  Slow test: ') + baseName + colors.yellow(` (${milliseconds(duration)})`));
+    }
+  }
+
+  epilogue(full: boolean) {
+    let skipped = 0;
+    let expected = 0;
+    const unexpected: Test[] = [];
+    const flaky: Test[] = [];
+
+    this.suite.findTest(test => {
+      switch (test.status()) {
+        case 'skipped': ++skipped; break;
+        case 'expected': ++expected; break;
+        case 'unexpected': unexpected.push(test); break;
+        case 'flaky': flaky.push(test); break;
+      }
+    });
+
+    if (full && unexpected.length) {
+      console.log('');
+      this._printFailures(unexpected);
+    }
+
+    this._printSlowTests();
+
+    console.log('');
+    if (unexpected.length) {
+      console.log(colors.red(`  ${unexpected.length} failed`));
+      this._printTestHeaders(unexpected);
+    }
+    if (flaky.length) {
+      console.log(colors.red(`  ${flaky.length} flaky`));
+      this._printTestHeaders(flaky);
+    }
+    if (skipped)
+      console.log(colors.yellow(`  ${skipped} skipped`));
+    if (expected)
+      console.log(colors.green(`  ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
+    if (this.timeout)
+      console.log(colors.red(`  Timed out waiting ${this.timeout / 1000}s for the entire test run`));
+  }
+
+  private _printTestHeaders(tests: Test[]) {
+    tests.forEach(test => {
+      console.log(formatTestHeader(this.config, test, '    '));
+    });
+  }
+
+  private _printFailures(failures: Test[]) {
+    failures.forEach((test, index) => {
+      console.log(formatFailure(this.config, test, index + 1));
+    });
+  }
+
+  hasResultWithStatus(test: Test, status: TestStatus): boolean {
+    return !!test.results.find(r => r.status === status);
+  }
+
+  willRetry(test: Test, result: TestResult): boolean {
+    return result.status !== 'passed' && result.status !== test.expectedStatus && test.results.length <= test.retries;
+  }
+}
+
+export function formatFailure(config: FullConfig, test: Test, index?: number): string {
+  const tokens: string[] = [];
+  tokens.push(formatTestHeader(config, test, '  ', index));
+  for (const result of test.results) {
+    if (result.status === 'passed')
+      continue;
+    tokens.push(formatFailedResult(test, result));
+  }
+  tokens.push('');
+  return tokens.join('\n');
+}
+
+export function formatTestTitle(config: FullConfig, test: Test): string {
+  const spec = test.spec;
+  let relativePath = path.relative(config.rootDir, spec.file) || path.basename(spec.file);
+  relativePath += ':' + spec.line + ':' + spec.column;
+  return `${relativePath} › ${test.fullTitle()}`;
+}
+
+function formatTestHeader(config: FullConfig, test: Test, indent: string, index?: number): string {
+  const title = formatTestTitle(config, test);
+  const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : '';
+  const header = `${indent}${index ? index + ') ' : ''}${title}${passedUnexpectedlySuffix}`;
+  return colors.red(pad(header, '='));
+}
+
+function formatFailedResult(test: Test, result: TestResult): string {
+  const tokens: string[] = [];
+  if (result.retry)
+    tokens.push(colors.gray(pad(`\n    Retry #${result.retry}`, '-')));
+  if (result.status === 'timedOut') {
+    tokens.push('');
+    tokens.push(indent(colors.red(`Timeout of ${test.timeout}ms exceeded.`), '    '));
+  } else {
+    tokens.push(indent(formatError(result.error!, test.spec.file), '    '));
+  }
+  return tokens.join('\n');
+}
+
+function formatError(error: TestError, file?: string) {
+  const stack = error.stack;
+  const tokens = [];
+  if (stack) {
+    tokens.push('');
+    const message = error.message || '';
+    const messageLocation = stack.indexOf(message);
+    const preamble = stack.substring(0, messageLocation + message.length);
+    tokens.push(preamble);
+    const position = file ? positionInFile(stack, file) : null;
+    if (position) {
+      const source = fs.readFileSync(file!, 'utf8');
+      tokens.push('');
+      tokens.push(codeFrameColumns(source, {
+        start: position,
+      },
+      { highlightCode: colors.enabled }
+      ));
+    }
+    tokens.push('');
+    tokens.push(colors.dim(preamble.length > 0 ? stack.substring(preamble.length + 1) : stack));
+  } else {
+    tokens.push('');
+    tokens.push(error.value);
+  }
+  return tokens.join('\n');
+}
+
+function pad(line: string, char: string): string {
+  return line + ' ' + colors.gray(char.repeat(Math.max(0, 100 - line.length - 1)));
+}
+
+function indent(lines: string, tab: string) {
+  return lines.replace(/^(?=.+$)/gm, tab);
+}
+
+function positionInFile(stack: string, file: string): { column: number; line: number; } {
+  // Stack will have /private/var/folders instead of /var/folders on Mac.
+  file = fs.realpathSync(file);
+  for (const line of stack.split('\n')) {
+    const parsed = stackUtils.parseLine(line);
+    if (!parsed || !parsed.file)
+      continue;
+    if (path.resolve(process.cwd(), parsed.file) === file)
+      return {column: parsed.column || 0, line: parsed.line || 0};
+  }
+  return { column: 0, line: 0 };
+}
+
+function monotonicTime(): number {
+  const [seconds, nanoseconds] = process.hrtime();
+  return seconds * 1000 + (nanoseconds / 1000000 | 0);
+}
+
+const asciiRegex = new RegExp('[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))', 'g');
+export function stripAscii(str: string): string {
+  return str.replace(asciiRegex, '');
+}

src/test/reporters/dot.ts

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import colors from 'colors/safe';
+import { BaseReporter } from './base';
+import { Test, TestResult } from '../reporter';
+
+class DotReporter extends BaseReporter {
+  private _counter = 0;
+
+  onTestEnd(test: Test, result: TestResult) {
+    super.onTestEnd(test, result);
+    if (++this._counter === 81) {
+      process.stdout.write('\n');
+      return;
+    }
+    if (result.status === 'skipped') {
+      process.stdout.write(colors.yellow('°'));
+      return;
+    }
+    if (this.willRetry(test, result)) {
+      process.stdout.write(colors.gray('×'));
+      return;
+    }
+    switch (test.status()) {
+      case 'expected': process.stdout.write(colors.green('·')); break;
+      case 'unexpected': process.stdout.write(colors.red(test.results[test.results.length - 1].status === 'timedOut' ? 'T' : 'F')); break;
+      case 'flaky': process.stdout.write(colors.yellow('±')); break;
+    }
+  }
+
+  onTimeout(timeout: number) {
+    super.onTimeout(timeout);
+    this.onEnd();
+  }
+
+  onEnd() {
+    super.onEnd();
+    process.stdout.write('\n');
+    this.epilogue(true);
+  }
+}
+
+export default DotReporter;

src/test/reporters/empty.ts

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FullConfig, TestResult, Test, Suite, TestError, Reporter } from '../reporter';
+
+class EmptyReporter implements Reporter {
+  onBegin(config: FullConfig, suite: Suite) {}
+  onTestBegin(test: Test) {}
+  onStdOut(chunk: string | Buffer, test?: Test) {}
+  onStdErr(chunk: string | Buffer, test?: Test) {}
+  onTestEnd(test: Test, result: TestResult) {}
+  onTimeout(timeout: number) {}
+  onError(error: TestError) {}
+  onEnd() {}
+}
+
+export default EmptyReporter;

src/test/reporters/json.ts

@@ -0,0 +1,160 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import EmptyReporter from './empty';
+import { FullConfig, Test, Suite, Spec, TestResult, TestError } from '../reporter';
+
+interface SerializedSuite {
+  title: string;
+  file: string;
+  column: number;
+  line: number;
+  specs: ReturnType<JSONReporter['_serializeTestSpec']>[];
+  suites?: SerializedSuite[];
+}
+
+export type ReportFormat = ReturnType<JSONReporter['_serializeReport']>;
+
+function toPosixPath(aPath: string): string {
+  return aPath.split(path.sep).join(path.posix.sep);
+}
+
+class JSONReporter extends EmptyReporter {
+  config!: FullConfig;
+  suite!: Suite;
+  private _errors: TestError[] = [];
+  private _outputFile: string | undefined;
+
+  constructor(options: { outputFile?: string } = {}) {
+    super();
+    this._outputFile = options.outputFile;
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
+    this.config = config;
+    this.suite = suite;
+  }
+
+  onTimeout() {
+    this.onEnd();
+  }
+
+  onError(error: TestError): void {
+    this._errors.push(error);
+  }
+
+  onEnd() {
+    outputReport(this._serializeReport(), this._outputFile);
+  }
+
+  private _serializeReport() {
+    return {
+      config: {
+        ...this.config,
+        rootDir: toPosixPath(this.config.rootDir),
+        projects: this.config.projects.map(project => {
+          return {
+            outputDir: toPosixPath(project.outputDir),
+            repeatEach: project.repeatEach,
+            retries: project.retries,
+            metadata: project.metadata,
+            name: project.name,
+            testDir: toPosixPath(project.testDir),
+            testIgnore: serializePatterns(project.testIgnore),
+            testMatch: serializePatterns(project.testMatch),
+            timeout: project.timeout,
+          };
+        })
+      },
+      suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s),
+      errors: this._errors
+    };
+  }
+
+  private _serializeSuite(suite: Suite): null | SerializedSuite {
+    if (!suite.findSpec(test => true))
+      return null;
+    const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s) as SerializedSuite[];
+    return {
+      title: suite.title,
+      file: toPosixPath(path.relative(this.config.rootDir, suite.file)),
+      line: suite.line,
+      column: suite.column,
+      specs: suite.specs.map(test => this._serializeTestSpec(test)),
+      suites: suites.length ? suites : undefined,
+    };
+  }
+
+  private _serializeTestSpec(spec: Spec) {
+    return {
+      title: spec.title,
+      ok: spec.ok(),
+      tests: spec.tests.map(r => this._serializeTest(r)),
+      file: toPosixPath(path.relative(this.config.rootDir, spec.file)),
+      line: spec.line,
+      column: spec.column,
+    };
+  }
+
+  private _serializeTest(test: Test) {
+    return {
+      timeout: test.timeout,
+      annotations: test.annotations,
+      expectedStatus: test.expectedStatus,
+      projectName: test.projectName,
+      results: test.results.map(r => this._serializeTestResult(r)),
+    };
+  }
+
+  private _serializeTestResult(result: TestResult) {
+    return {
+      workerIndex: result.workerIndex,
+      status: result.status,
+      duration: result.duration,
+      error: result.error,
+      stdout: result.stdout.map(s => stdioEntry(s)),
+      stderr: result.stderr.map(s => stdioEntry(s)),
+      retry: result.retry,
+    };
+  }
+}
+
+function outputReport(report: ReportFormat, outputFile: string | undefined) {
+  const reportString = JSON.stringify(report, undefined, 2);
+  outputFile = outputFile || process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`];
+  if (outputFile) {
+    fs.mkdirSync(path.dirname(outputFile), { recursive: true });
+    fs.writeFileSync(outputFile, reportString);
+  } else {
+    console.log(reportString);
+  }
+}
+
+function stdioEntry(s: string | Buffer): any {
+  if (typeof s === 'string')
+    return { text: s };
+  return { buffer: s.toString('base64') };
+}
+
+function serializePatterns(patterns: string | RegExp | (string | RegExp)[]): string[] {
+  if (!Array.isArray(patterns))
+    patterns = [patterns];
+  return patterns.map(s => s.toString());
+}
+
+export default JSONReporter;

src/test/reporters/junit.ts

@@ -0,0 +1,199 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import EmptyReporter from './empty';
+import { FullConfig, Suite, Test } from '../reporter';
+import { monotonicTime } from '../util';
+import { formatFailure, formatTestTitle, stripAscii } from './base';
+
+class JUnitReporter extends EmptyReporter {
+  private config!: FullConfig;
+  private suite!: Suite;
+  private timestamp!: number;
+  private startTime!: number;
+  private totalTests = 0;
+  private totalFailures = 0;
+  private totalSkipped = 0;
+  private outputFile: string | undefined;
+  private stripANSIControlSequences = false;
+
+  constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean } = {}) {
+    super();
+    this.outputFile = options.outputFile;
+    this.stripANSIControlSequences = options.stripANSIControlSequences || false;
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
+    this.config = config;
+    this.suite = suite;
+    this.timestamp = Date.now();
+    this.startTime = monotonicTime();
+  }
+
+  onEnd() {
+    const duration = monotonicTime() - this.startTime;
+    const children: XMLEntry[] = [];
+    for (const suite of this.suite.suites)
+      children.push(this._buildTestSuite(suite));
+    const tokens: string[] = [];
+
+    const self = this;
+    const root: XMLEntry = {
+      name: 'testsuites',
+      attributes: {
+        id: process.env[`PLAYWRIGHT_JUNIT_SUITE_ID`] || '',
+        name: process.env[`PLAYWRIGHT_JUNIT_SUITE_NAME`] || '',
+        tests: self.totalTests,
+        failures: self.totalFailures,
+        skipped: self.totalSkipped,
+        errors: 0,
+        time: duration / 1000
+      },
+      children
+    };
+
+    serializeXML(root, tokens, this.stripANSIControlSequences);
+    const reportString = tokens.join('\n');
+    const outputFile = this.outputFile || process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`];
+    if (outputFile) {
+      fs.mkdirSync(path.dirname(outputFile), { recursive: true });
+      fs.writeFileSync(outputFile, reportString);
+    } else {
+      console.log(reportString);
+    }
+  }
+
+  private _buildTestSuite(suite: Suite): XMLEntry {
+    let tests = 0;
+    let skipped = 0;
+    let failures = 0;
+    let duration = 0;
+    const children: XMLEntry[] = [];
+
+    suite.findTest(test => {
+      ++tests;
+      if (test.skipped)
+        ++skipped;
+      if (!test.ok())
+        ++failures;
+      for (const result of test.results)
+        duration += result.duration;
+      this._addTestCase(test, children);
+    });
+    this.totalTests += tests;
+    this.totalSkipped += skipped;
+    this.totalFailures += failures;
+
+    const entry: XMLEntry = {
+      name: 'testsuite',
+      attributes: {
+        name: path.relative(this.config.rootDir, suite.file),
+        timestamp: this.timestamp,
+        hostname: '',
+        tests,
+        failures,
+        skipped,
+        time: duration / 1000,
+        errors: 0,
+      },
+      children
+    };
+
+    return entry;
+  }
+
+  private _addTestCase(test: Test, entries: XMLEntry[]) {
+    const entry = {
+      name: 'testcase',
+      attributes: {
+        name: test.spec.fullTitle(),
+        classname: formatTestTitle(this.config, test),
+        time: (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000
+      },
+      children: [] as XMLEntry[]
+    };
+    entries.push(entry);
+
+    if (test.skipped) {
+      entry.children.push({ name: 'skipped'});
+      return;
+    }
+
+    if (!test.ok()) {
+      entry.children.push({
+        name: 'failure',
+        attributes: {
+          message: `${path.basename(test.spec.file)}:${test.spec.line}:${test.spec.column} ${test.spec.title}`,
+          type: 'FAILURE',
+        },
+        text: stripAscii(formatFailure(this.config, test))
+      });
+    }
+    for (const result of test.results) {
+      for (const stdout of result.stdout) {
+        entries.push({
+          name: 'system-out',
+          text: stdout.toString()
+        });
+      }
+
+      for (const stderr of result.stderr) {
+        entries.push({
+          name: 'system-err',
+          text: stderr.toString()
+        });
+      }
+    }
+  }
+}
+
+type XMLEntry = {
+  name: string;
+  attributes?: { [name: string]: string | number | boolean };
+  children?: XMLEntry[];
+  text?: string;
+};
+
+function serializeXML(entry: XMLEntry, tokens: string[], stripANSIControlSequences: boolean) {
+  const attrs: string[] = [];
+  for (const [name, value] of Object.entries(entry.attributes || {}))
+    attrs.push(`${name}="${escape(String(value), stripANSIControlSequences, false)}"`);
+  tokens.push(`<${entry.name}${attrs.length ? ' ' : ''}${attrs.join(' ')}>`);
+  for (const child of entry.children || [])
+    serializeXML(child, tokens, stripANSIControlSequences);
+  if (entry.text)
+    tokens.push(escape(entry.text, stripANSIControlSequences, true));
+  tokens.push(`</${entry.name}>`);
+}
+
+// See https://en.wikipedia.org/wiki/Valid_characters_in_XML
+const discouragedXMLCharacters = /[\u0001-\u0008\u000b-\u000c\u000e-\u001f\u007f-\u0084\u0086-\u009f]/g;
+const ansiControlSequence = new RegExp('[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))', 'g');
+
+function escape(text: string, stripANSIControlSequences: boolean, isCharacterData: boolean): string {
+  if (stripANSIControlSequences)
+    text = text.replace(ansiControlSequence, '');
+  const escapeRe = isCharacterData ? /[&<]/g : /[&"<>]/g;
+  text = text.replace(escapeRe, c => ({ '&': '&amp;', '"': '&quot;', '<': '&lt;', '>': '&gt;' }[c]!));
+  if (isCharacterData)
+    text = text.replace(/]]>/g, ']]&gt;');
+  text = text.replace(discouragedXMLCharacters, '');
+  return text;
+}
+
+export default JUnitReporter;

src/test/reporters/line.ts

@@ -0,0 +1,74 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import colors from 'colors/safe';
+import { BaseReporter, formatFailure, formatTestTitle } from './base';
+import { FullConfig, Test, Suite, TestResult } from '../reporter';
+
+class LineReporter extends BaseReporter {
+  private _total = 0;
+  private _current = 0;
+  private _failures = 0;
+  private _lastTest: Test | undefined;
+
+  onBegin(config: FullConfig, suite: Suite) {
+    super.onBegin(config, suite);
+    this._total = suite.totalTestCount();
+    console.log();
+  }
+
+  onStdOut(chunk: string | Buffer, test?: Test) {
+    this._dumpToStdio(test, chunk, process.stdout);
+  }
+
+  onStdErr(chunk: string | Buffer, test?: Test) {
+    this._dumpToStdio(test, chunk, process.stderr);
+  }
+
+  private _dumpToStdio(test: Test | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
+    if (this.config.quiet)
+      return;
+    stream.write(`\u001B[1A\u001B[2K`);
+    if (test && this._lastTest !== test) {
+      // Write new header for the output.
+      stream.write(colors.gray(formatTestTitle(this.config, test) + `\n`));
+      this._lastTest = test;
+    }
+
+    stream.write(chunk);
+    console.log();
+  }
+
+  onTestEnd(test: Test, result: TestResult) {
+    super.onTestEnd(test, result);
+    const width = process.stdout.columns! - 1;
+    const title = `[${++this._current}/${this._total}] ${formatTestTitle(this.config, test)}`.substring(0, width);
+    process.stdout.write(`\u001B[1A\u001B[2K${title}\n`);
+    if (!this.willRetry(test, result) && !test.ok()) {
+      process.stdout.write(`\u001B[1A\u001B[2K`);
+      console.log(formatFailure(this.config, test, ++this._failures));
+      console.log();
+    }
+  }
+
+  onEnd() {
+    process.stdout.write(`\u001B[1A\u001B[2K`);
+    super.onEnd();
+    this.epilogue(false);
+  }
+}
+
+export default LineReporter;

src/test/reporters/list.ts

@@ -0,0 +1,111 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import colors from 'colors/safe';
+// @ts-ignore
+import milliseconds from 'ms';
+import { BaseReporter, formatTestTitle } from './base';
+import { FullConfig, Suite, Test, TestResult } from '../reporter';
+
+class ListReporter extends BaseReporter {
+  private _failure = 0;
+  private _lastRow = 0;
+  private _testRows = new Map<Test, number>();
+  private _needNewLine = false;
+
+  onBegin(config: FullConfig, suite: Suite) {
+    super.onBegin(config, suite);
+    console.log();
+  }
+
+  onTestBegin(test: Test) {
+    super.onTestBegin(test);
+    if (process.stdout.isTTY) {
+      if (this._needNewLine) {
+        this._needNewLine = false;
+        process.stdout.write('\n');
+        this._lastRow++;
+      }
+      process.stdout.write('    ' + colors.gray(formatTestTitle(this.config, test) + ': ') + '\n');
+    }
+    this._testRows.set(test, this._lastRow++);
+  }
+
+  onStdOut(chunk: string | Buffer, test?: Test) {
+    this._dumpToStdio(test, chunk, process.stdout);
+  }
+
+  onStdErr(chunk: string | Buffer, test?: Test) {
+    this._dumpToStdio(test, chunk, process.stdout);
+  }
+
+  private _dumpToStdio(test: Test | undefined, chunk: string | Buffer, stream: NodeJS.WriteStream) {
+    if (this.config.quiet)
+      return;
+    const text = chunk.toString('utf-8');
+    this._needNewLine = text[text.length - 1] !== '\n';
+    if (process.stdout.isTTY) {
+      const newLineCount = text.split('\n').length - 1;
+      this._lastRow += newLineCount;
+    }
+    stream.write(chunk);
+  }
+
+  onTestEnd(test: Test, result: TestResult) {
+    super.onTestEnd(test, result);
+
+    const duration = colors.dim(` (${milliseconds(result.duration)})`);
+    const title = formatTestTitle(this.config, test);
+    let text = '';
+    if (result.status === 'skipped') {
+      text = colors.green('  - ') + colors.cyan(title);
+    } else {
+      const statusMark = result.status === 'passed' ? '  ✓ ' : '  x ';
+      if (result.status === test.expectedStatus)
+        text = '\u001b[2K\u001b[0G' + colors.green(statusMark) + colors.gray(title) + duration;
+      else
+        text = '\u001b[2K\u001b[0G' + colors.red(`${statusMark}${++this._failure}) ` + title) + duration;
+    }
+
+    const testRow = this._testRows.get(test)!;
+    // Go up if needed
+    if (process.stdout.isTTY && testRow !== this._lastRow)
+      process.stdout.write(`\u001B[${this._lastRow - testRow}A`);
+    // Erase line
+    if (process.stdout.isTTY)
+      process.stdout.write('\u001B[2K');
+    if (!process.stdout.isTTY && this._needNewLine) {
+      this._needNewLine = false;
+      process.stdout.write('\n');
+    }
+    process.stdout.write(text);
+    // Go down if needed.
+    if (testRow !== this._lastRow) {
+      if (process.stdout.isTTY)
+        process.stdout.write(`\u001B[${this._lastRow - testRow}E`);
+      else
+        process.stdout.write('\n');
+    }
+  }
+
+  onEnd() {
+    super.onEnd();
+    process.stdout.write('\n');
+    this.epilogue(true);
+  }
+}
+
+export default ListReporter;

src/test/reporters/multiplexer.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FullConfig, Suite, Test, TestError, TestResult, Reporter } from '../reporter';
+
+export class Multiplexer implements Reporter {
+  private _reporters: Reporter[];
+
+  constructor(reporters: Reporter[]) {
+    this._reporters = reporters;
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
+    for (const reporter of this._reporters)
+      reporter.onBegin(config, suite);
+  }
+
+  onTestBegin(test: Test) {
+    for (const reporter of this._reporters)
+      reporter.onTestBegin(test);
+  }
+
+  onStdOut(chunk: string | Buffer, test?: Test) {
+    for (const reporter of this._reporters)
+      reporter.onStdOut(chunk, test);
+  }
+
+  onStdErr(chunk: string | Buffer, test?: Test) {
+    for (const reporter of this._reporters)
+      reporter.onStdErr(chunk, test);
+  }
+
+  onTestEnd(test: Test, result: TestResult) {
+    for (const reporter of this._reporters)
+      reporter.onTestEnd(test, result);
+  }
+
+  onTimeout(timeout: number) {
+    for (const reporter of this._reporters)
+      reporter.onTimeout(timeout);
+  }
+
+  onEnd() {
+    for (const reporter of this._reporters)
+      reporter.onEnd();
+  }
+
+  onError(error: TestError) {
+    for (const reporter of this._reporters)
+      reporter.onError(error);
+  }
+}

src/test/runner.ts

@@ -0,0 +1,319 @@
+/**
+ * Copyright 2019 Google Inc. All rights reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import rimraf from 'rimraf';
+import * as fs from 'fs';
+import * as path from 'path';
+import { promisify } from 'util';
+import { Dispatcher } from './dispatcher';
+import { createMatcher, monotonicTime, raceAgainstDeadline } from './util';
+import { Suite } from './test';
+import { Loader } from './loader';
+import { Reporter } from './reporter';
+import { Multiplexer } from './reporters/multiplexer';
+import DotReporter from './reporters/dot';
+import LineReporter from './reporters/line';
+import ListReporter from './reporters/list';
+import JSONReporter from './reporters/json';
+import JUnitReporter from './reporters/junit';
+import EmptyReporter from './reporters/empty';
+import { ProjectImpl } from './project';
+import { Minimatch } from 'minimatch';
+import { Config } from './types';
+
+const removeFolderAsync = promisify(rimraf);
+const readDirAsync = promisify(fs.readdir);
+const readFileAsync = promisify(fs.readFile);
+
+type RunResult = 'passed' | 'failed' | 'sigint' | 'forbid-only' | 'no-tests' | 'timedout';
+
+export class Runner {
+  private _loader: Loader;
+  private _reporter!: Reporter;
+  private _didBegin = false;
+
+  constructor(defaultConfig: Config, configOverrides: Config) {
+    this._loader = new Loader(defaultConfig, configOverrides);
+  }
+
+  private _createReporter() {
+    const reporters: Reporter[] = [];
+    const defaultReporters = {
+      dot: DotReporter,
+      line: LineReporter,
+      list: ListReporter,
+      json: JSONReporter,
+      junit: JUnitReporter,
+      null: EmptyReporter,
+    };
+    for (const r of this._loader.fullConfig().reporter) {
+      const [name, arg] = r;
+      if (name in defaultReporters) {
+        reporters.push(new defaultReporters[name as keyof typeof defaultReporters](arg));
+      } else {
+        const reporterConstructor = this._loader.loadReporter(name);
+        reporters.push(new reporterConstructor(arg));
+      }
+    }
+    return new Multiplexer(reporters);
+  }
+
+  loadConfigFile(file: string): Config {
+    return this._loader.loadConfigFile(file);
+  }
+
+  loadEmptyConfig(rootDir: string) {
+    this._loader.loadEmptyConfig(rootDir);
+  }
+
+  async run(list: boolean, testFileReFilters: RegExp[], projectName?: string): Promise<RunResult> {
+    this._reporter = this._createReporter();
+    const config = this._loader.fullConfig();
+    const globalDeadline = config.globalTimeout ? config.globalTimeout + monotonicTime() : undefined;
+    const { result, timedOut } = await raceAgainstDeadline(this._run(list, testFileReFilters, projectName), globalDeadline);
+    if (timedOut) {
+      if (!this._didBegin)
+        this._reporter.onBegin(config, new Suite(''));
+      this._reporter.onTimeout(config.globalTimeout);
+      await this._flushOutput();
+      return 'failed';
+    }
+    if (result === 'forbid-only') {
+      console.error('=====================================');
+      console.error(' --forbid-only found a focused test.');
+      console.error('=====================================');
+    } else if (result === 'no-tests') {
+      console.error('=================');
+      console.error(' no tests found.');
+      console.error('=================');
+    }
+    await this._flushOutput();
+    return result!;
+  }
+
+  async _flushOutput() {
+    // Calling process.exit() might truncate large stdout/stderr output.
+    // See https://github.com/nodejs/node/issues/6456.
+    //
+    // We can use writableNeedDrain to workaround this, but it is only available
+    // since node v15.2.0.
+    // See https://nodejs.org/api/stream.html#stream_writable_writableneeddrain.
+    if ((process.stdout as any).writableNeedDrain)
+      await new Promise(f => process.stdout.on('drain', f));
+    if ((process.stderr as any).writableNeedDrain)
+      await new Promise(f => process.stderr.on('drain', f));
+  }
+
+  async _run(list: boolean, testFileReFilters: RegExp[], projectName?: string): Promise<RunResult> {
+    const testFileFilter = testFileReFilters.length ? createMatcher(testFileReFilters) : () => true;
+    const config = this._loader.fullConfig();
+
+    const projects = this._loader.projects().filter(project => {
+      return !projectName || project.config.name.toLocaleLowerCase() === projectName.toLocaleLowerCase();
+    });
+    if (projectName && !projects.length) {
+      const names = this._loader.projects().map(p => p.config.name).filter(name => !!name);
+      if (!names.length)
+        throw new Error(`No named projects are specified in the configuration file`);
+      throw new Error(`Project "${projectName}" not found. Available named projects: ${names.map(name => `"${name}"`).join(', ')}`);
+    }
+
+    const files = new Map<ProjectImpl, string[]>();
+    const allTestFiles = new Set<string>();
+    for (const project of projects) {
+      const testDir = project.config.testDir;
+      if (!fs.existsSync(testDir))
+        throw new Error(`${testDir} does not exist`);
+      if (!fs.statSync(testDir).isDirectory())
+        throw new Error(`${testDir} is not a directory`);
+      const allFiles = await collectFiles(project.config.testDir);
+      const testMatch = createMatcher(project.config.testMatch);
+      const testIgnore = createMatcher(project.config.testIgnore);
+      const testFiles = allFiles.filter(file => !testIgnore(file) && testMatch(file) && testFileFilter(file));
+      files.set(project, testFiles);
+      testFiles.forEach(file => allTestFiles.add(file));
+    }
+
+    let globalSetupResult: any;
+    if (config.globalSetup)
+      globalSetupResult = await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup')(this._loader.fullConfig());
+    try {
+      for (const file of allTestFiles)
+        this._loader.loadTestFile(file);
+
+      const rootSuite = new Suite('');
+      for (const fileSuite of this._loader.fileSuites().values())
+        rootSuite._addSuite(fileSuite);
+      if (config.forbidOnly && rootSuite._hasOnly())
+        return 'forbid-only';
+      filterOnly(rootSuite);
+
+      const fileSuites = new Map<string, Suite>();
+      for (const fileSuite of rootSuite.suites)
+        fileSuites.set(fileSuite.file, fileSuite);
+
+      const outputDirs = new Set<string>();
+      const grepMatcher = createMatcher(config.grep);
+      for (const project of projects) {
+        for (const file of files.get(project)!) {
+          const fileSuite = fileSuites.get(file);
+          if (!fileSuite)
+            continue;
+          for (const spec of fileSuite._allSpecs()) {
+            if (grepMatcher(spec._testFullTitle(project.config.name)))
+              project.generateTests(spec);
+          }
+        }
+        outputDirs.add(project.config.outputDir);
+      }
+
+      const total = rootSuite.totalTestCount();
+      if (!total)
+        return 'no-tests';
+
+      await Promise.all(Array.from(outputDirs).map(outputDir => removeFolderAsync(outputDir).catch(e => {})));
+
+      let sigint = false;
+      let sigintCallback: () => void;
+      const sigIntPromise = new Promise<void>(f => sigintCallback = f);
+      const sigintHandler = () => {
+        // We remove handler so that double Ctrl+C immediately kills the runner,
+        // for the case where our shutdown takes a lot of time or is buggy.
+        // Removing the handler synchronously sometimes triggers the default handler
+        // that exits the process, so we remove asynchronously.
+        setTimeout(() => process.off('SIGINT', sigintHandler), 0);
+        sigint = true;
+        sigintCallback();
+      };
+      process.on('SIGINT', sigintHandler);
+
+      if (process.stdout.isTTY) {
+        const workers = new Set();
+        rootSuite.findTest(test => {
+          workers.add(test.spec.file + test._workerHash);
+        });
+        console.log();
+        const jobs = Math.min(config.workers, workers.size);
+        const shard = config.shard;
+        const shardDetails = shard ? `, shard ${shard.current + 1} of ${shard.total}` : '';
+        console.log(`Running ${total} test${total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}${shardDetails}`);
+      }
+
+      this._reporter.onBegin(config, rootSuite);
+      this._didBegin = true;
+      let hasWorkerErrors = false;
+      if (!list) {
+        const dispatcher = new Dispatcher(this._loader, rootSuite, this._reporter);
+        await Promise.race([dispatcher.run(), sigIntPromise]);
+        await dispatcher.stop();
+        hasWorkerErrors = dispatcher.hasWorkerErrors();
+      }
+      this._reporter.onEnd();
+
+      if (sigint)
+        return 'sigint';
+      return hasWorkerErrors || rootSuite.findSpec(spec => !spec.ok()) ? 'failed' : 'passed';
+    } finally {
+      if (globalSetupResult && typeof globalSetupResult === 'function')
+        await globalSetupResult(this._loader.fullConfig());
+      if (config.globalTeardown)
+        await this._loader.loadGlobalHook(config.globalTeardown, 'globalTeardown')(this._loader.fullConfig());
+    }
+  }
+}
+
+function filterOnly(suite: Suite) {
+  const onlySuites = suite.suites.filter(child => filterOnly(child) || child._only);
+  const onlyTests = suite.specs.filter(spec => spec._only);
+  const onlyEntries = new Set([...onlySuites, ...onlyTests]);
+  if (onlyEntries.size) {
+    suite.suites = onlySuites;
+    suite.specs = onlyTests;
+    suite._entries = suite._entries.filter(e => onlyEntries.has(e)); // Preserve the order.
+    return true;
+  }
+  return false;
+}
+
+async function collectFiles(testDir: string): Promise<string[]> {
+  type Rule = {
+    dir: string;
+    negate: boolean;
+    match: (s: string, partial?: boolean) => boolean
+  };
+  type IgnoreStatus = 'ignored' | 'included' | 'ignored-but-recurse';
+
+  const checkIgnores = (entryPath: string, rules: Rule[], isDirectory: boolean, parentStatus: IgnoreStatus) => {
+    let status = parentStatus;
+    for (const rule of rules) {
+      const ruleIncludes = rule.negate;
+      if ((status === 'included') === ruleIncludes)
+        continue;
+      const relative = path.relative(rule.dir, entryPath);
+      if (rule.match('/' + relative) || rule.match(relative)) {
+        // Matches "/dir/file" or "dir/file"
+        status = ruleIncludes ? 'included' : 'ignored';
+      } else if (isDirectory && (rule.match('/' + relative + '/') || rule.match(relative + '/'))) {
+        // Matches "/dir/subdir/" or "dir/subdir/" for directories.
+        status = ruleIncludes ? 'included' : 'ignored';
+      } else if (isDirectory && ruleIncludes && (rule.match('/' + relative, true) || rule.match(relative, true))) {
+        // Matches "/dir/donotskip/" when "/dir" is excluded, but "!/dir/donotskip/file" is included.
+        status = 'ignored-but-recurse';
+      }
+    }
+    return status;
+  };
+
+  const files: string[] = [];
+
+  const visit = async (dir: string, rules: Rule[], status: IgnoreStatus) => {
+    const entries = await readDirAsync(dir, { withFileTypes: true });
+    entries.sort((a, b) => a.name.localeCompare(b.name));
+
+    const gitignore = entries.find(e => e.isFile() && e.name === '.gitignore');
+    if (gitignore) {
+      const content = await readFileAsync(path.join(dir, gitignore.name), 'utf8');
+      const newRules: Rule[] = content.split(/\r?\n/).map(s => {
+        s = s.trim();
+        if (!s)
+          return;
+        // Use flipNegate, because we handle negation ourselves.
+        const rule = new Minimatch(s, { matchBase: true, dot: true, flipNegate: true }) as any;
+        if (rule.comment)
+          return;
+        rule.dir = dir;
+        return rule;
+      }).filter(rule => !!rule);
+      rules = [...rules, ...newRules];
+    }
+
+    for (const entry of entries) {
+      if (entry === gitignore || entry.name === '.' || entry.name === '..')
+        continue;
+      if (entry.isDirectory() && entry.name === 'node_modules')
+        continue;
+      const entryPath = path.join(dir, entry.name);
+      const entryStatus = checkIgnores(entryPath, rules, entry.isDirectory(), status);
+      if (entry.isDirectory() && entryStatus !== 'ignored')
+        await visit(entryPath, rules, entryStatus);
+      else if (entry.isFile() && entryStatus === 'included')
+        files.push(entryPath);
+    }
+  };
+  await visit(testDir, [], 'included');
+  return files;
+}

src/test/test.ts

@@ -0,0 +1,223 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as reporterTypes from './reporter';
+import type { TestTypeImpl } from './testType';
+import { Location } from './types';
+
+class Base {
+  title: string;
+  file: string = '';
+  line: number = 0;
+  column: number = 0;
+  parent?: Suite;
+
+  _only = false;
+
+  constructor(title: string) {
+    this.title = title;
+  }
+
+  titlePath(): string[] {
+    if (!this.parent)
+      return [];
+    if (!this.title)
+      return this.parent.titlePath();
+    return [...this.parent.titlePath(), this.title];
+  }
+}
+
+export class Spec extends Base implements reporterTypes.Spec {
+  suite!: Suite;
+  fn: Function;
+  tests: Test[] = [];
+  _ordinalInFile: number;
+  _testType: TestTypeImpl;
+
+  constructor(title: string, fn: Function, ordinalInFile: number, testType: TestTypeImpl) {
+    super(title);
+    this.fn = fn;
+    this._ordinalInFile = ordinalInFile;
+    this._testType = testType;
+  }
+
+  ok(): boolean {
+    return !this.tests.find(r => !r.ok());
+  }
+
+  fullTitle(): string {
+    return this.titlePath().join(' ');
+  }
+
+  _testFullTitle(projectName: string) {
+    return (projectName ? `[${projectName}] ` : '') + this.fullTitle();
+  }
+}
+
+export class Suite extends Base implements reporterTypes.Suite {
+  suites: Suite[] = [];
+  specs: Spec[] = [];
+  _fixtureOverrides: any = {};
+  _entries: (Suite | Spec)[] = [];
+  _hooks: {
+    type: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll',
+    fn: Function,
+    location: Location,
+  } [] = [];
+
+  _addSpec(spec: Spec) {
+    spec.parent = this;
+    spec.suite = this;
+    this.specs.push(spec);
+    this._entries.push(spec);
+  }
+
+  _addSuite(suite: Suite) {
+    suite.parent = this;
+    this.suites.push(suite);
+    this._entries.push(suite);
+  }
+
+  findTest(fn: (test: Test) => boolean | void): boolean {
+    for (const entry of this._entries) {
+      if (entry instanceof Suite) {
+        if (entry.findTest(fn))
+          return true;
+      } else {
+        for (const test of entry.tests) {
+          if (fn(test))
+            return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  findSpec(fn: (spec: Spec) => boolean | void): boolean {
+    for (const entry of this._entries) {
+      if (entry instanceof Suite) {
+        if (entry.findSpec(fn))
+          return true;
+      } else {
+        if (fn(entry))
+          return true;
+      }
+    }
+    return false;
+  }
+
+  findSuite(fn: (suite: Suite) => boolean | void): boolean {
+    if (fn(this))
+      return true;
+    for (const suite of this.suites) {
+      if (suite.findSuite(fn))
+        return true;
+    }
+    return false;
+  }
+
+  totalTestCount(): number {
+    let total = 0;
+    for (const suite of this.suites)
+      total += suite.totalTestCount();
+    for (const spec of this.specs)
+      total += spec.tests.length;
+    return total;
+  }
+
+  _allSpecs(): Spec[] {
+    const result: Spec[] = [];
+    this.findSpec(test => { result.push(test); });
+    return result;
+  }
+
+  _hasOnly(): boolean {
+    if (this._only)
+      return true;
+    if (this.suites.find(suite => suite._hasOnly()))
+      return true;
+    if (this.specs.find(spec => spec._only))
+      return true;
+    return false;
+  }
+
+  _buildFixtureOverrides(): any {
+    return this.parent ? { ...this.parent._buildFixtureOverrides(), ...this._fixtureOverrides } : this._fixtureOverrides;
+  }
+}
+
+export class Test implements reporterTypes.Test {
+  spec: Spec;
+  results: reporterTypes.TestResult[] = [];
+
+  skipped = false;
+  expectedStatus: reporterTypes.TestStatus = 'passed';
+  timeout = 0;
+  annotations: { type: string, description?: string }[] = [];
+  projectName = '';
+  retries = 0;
+
+  _id = '';
+  _repeatEachIndex = 0;
+  _projectIndex = 0;
+  _workerHash = '';
+
+  constructor(spec: Spec) {
+    this.spec = spec;
+  }
+
+  status(): 'skipped' | 'expected' | 'unexpected' | 'flaky' {
+    if (this.skipped)
+      return 'skipped';
+    // List mode bail out.
+    if (!this.results.length)
+      return 'skipped';
+    if (this.results.length === 1 && this.expectedStatus === this.results[0].status)
+      return 'expected';
+    let hasPassedResults = false;
+    for (const result of this.results) {
+      // Missing status is Ok when running in shards mode.
+      if (!result.status)
+        return 'skipped';
+      if (result.status === this.expectedStatus)
+        hasPassedResults = true;
+    }
+    if (hasPassedResults)
+      return 'flaky';
+    return 'unexpected';
+  }
+
+  ok(): boolean {
+    const status = this.status();
+    return status === 'expected' || status === 'flaky' || status === 'skipped';
+  }
+
+  fullTitle(): string {
+    return this.spec._testFullTitle(this.projectName);
+  }
+
+  _appendTestResult(): reporterTypes.TestResult {
+    const result: reporterTypes.TestResult = {
+      retry: this.results.length,
+      workerIndex: 0,
+      duration: 0,
+      stdout: [],
+      stderr: [],
+    };
+    this.results.push(result);
+    return result;
+  }
+}

src/test/testType.ts

@@ -0,0 +1,161 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { expect } from './expect';
+import { currentlyLoadingFileSuite, currentTestInfo, setCurrentlyLoadingFileSuite } from './globals';
+import { Spec, Suite } from './test';
+import { callLocation, errorWithCallLocation } from './util';
+import { Fixtures, FixturesWithLocation, Location, TestInfo, TestType } from './types';
+import { inheritFixtureParameterNames } from './fixtures';
+
+Error.stackTraceLimit = 15;
+
+const countByFile = new Map<string, number>();
+
+export class DeclaredFixtures {
+  testType!: TestTypeImpl;
+  location!: Location;
+}
+
+export class TestTypeImpl {
+  readonly fixtures: (FixturesWithLocation | DeclaredFixtures)[];
+  readonly test: TestType<any, any>;
+
+  constructor(fixtures: (FixturesWithLocation | DeclaredFixtures)[]) {
+    this.fixtures = fixtures;
+
+    const test: any = this._spec.bind(this, 'default');
+    test.expect = expect;
+    test.only = this._spec.bind(this, 'only');
+    test.describe = this._describe.bind(this, 'default');
+    test.describe.only = this._describe.bind(this, 'only');
+    test.beforeEach = this._hook.bind(this, 'beforeEach');
+    test.afterEach = this._hook.bind(this, 'afterEach');
+    test.beforeAll = this._hook.bind(this, 'beforeAll');
+    test.afterAll = this._hook.bind(this, 'afterAll');
+    test.skip = this._modifier.bind(this, 'skip');
+    test.fixme = this._modifier.bind(this, 'fixme');
+    test.fail = this._modifier.bind(this, 'fail');
+    test.slow = this._modifier.bind(this, 'slow');
+    test.setTimeout = this._setTimeout.bind(this);
+    test.use = this._use.bind(this);
+    test.extend = this._extend.bind(this);
+    test.declare = this._declare.bind(this);
+    this.test = test;
+  }
+
+  private _spec(type: 'default' | 'only', title: string, fn: Function) {
+    const suite = currentlyLoadingFileSuite();
+    if (!suite)
+      throw errorWithCallLocation(`test() can only be called in a test file`);
+    const location = callLocation(suite.file);
+
+    const ordinalInFile = countByFile.get(suite.file) || 0;
+    countByFile.set(location.file, ordinalInFile + 1);
+
+    const spec = new Spec(title, fn, ordinalInFile, this);
+    spec.file = location.file;
+    spec.line = location.line;
+    spec.column = location.column;
+    suite._addSpec(spec);
+
+    if (type === 'only')
+      spec._only = true;
+  }
+
+  private _describe(type: 'default' | 'only', title: string, fn: Function) {
+    const suite = currentlyLoadingFileSuite();
+    if (!suite)
+      throw errorWithCallLocation(`describe() can only be called in a test file`);
+    const location = callLocation(suite.file);
+
+    const child = new Suite(title);
+    child.file = suite.file;
+    child.line = location.line;
+    child.column = location.column;
+    suite._addSuite(child);
+
+    if (type === 'only')
+      child._only = true;
+
+    setCurrentlyLoadingFileSuite(child);
+    fn();
+    setCurrentlyLoadingFileSuite(suite);
+  }
+
+  private _hook(name: 'beforeEach' | 'afterEach' | 'beforeAll' | 'afterAll', fn: Function) {
+    const suite = currentlyLoadingFileSuite();
+    if (!suite)
+      throw errorWithCallLocation(`${name} hook can only be called in a test file`);
+    suite._hooks.push({ type: name, fn, location: callLocation() });
+  }
+
+  private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', ...modiferAgs: [arg?: any | Function, description?: string]) {
+    const suite = currentlyLoadingFileSuite();
+    if (suite) {
+      const location = callLocation();
+      if (typeof modiferAgs[0] === 'function') {
+        const [conditionFn, description] = modiferAgs;
+        const fn = (args: any, testInfo: TestInfo) => testInfo[type](conditionFn(args), description!);
+        inheritFixtureParameterNames(conditionFn, fn, location);
+        suite._hooks.unshift({ type: 'beforeEach', fn, location });
+      } else {
+        const fn = ({}: any, testInfo: TestInfo) => testInfo[type](...modiferAgs as [any, any]);
+        suite._hooks.unshift({ type: 'beforeEach', fn, location });
+      }
+      return;
+    }
+
+    const testInfo = currentTestInfo();
+    if (!testInfo)
+      throw new Error(`test.${type}() can only be called inside test, describe block or fixture`);
+    if (typeof modiferAgs[0] === 'function')
+      throw new Error(`test.${type}() with a function can only be called inside describe block`);
+    testInfo[type](...modiferAgs as [any, any]);
+  }
+
+  private _setTimeout(timeout: number) {
+    const testInfo = currentTestInfo();
+    if (!testInfo)
+      throw new Error(`test.setTimeout() can only be called inside test or fixture`);
+    testInfo.setTimeout(timeout);
+  }
+
+  private _use(fixtures: Fixtures) {
+    const suite = currentlyLoadingFileSuite();
+    if (!suite)
+      throw errorWithCallLocation(`test.use() can only be called in a test file`);
+    suite._fixtureOverrides = { ...suite._fixtureOverrides, ...fixtures };
+  }
+
+  private _extend(fixtures: Fixtures) {
+    const fixturesWithLocation = {
+      fixtures,
+      location: callLocation(),
+    };
+    return new TestTypeImpl([...this.fixtures, fixturesWithLocation]).test;
+  }
+
+  private _declare() {
+    const declared = new DeclaredFixtures();
+    declared.location = callLocation();
+    const child = new TestTypeImpl([...this.fixtures, declared]);
+    declared.testType = child;
+    return child.test;
+  }
+}
+
+export const rootTestType = new TestTypeImpl([]);

src/test/transform.ts

@@ -0,0 +1,99 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as crypto from 'crypto';
+import * as os from 'os';
+import * as path from 'path';
+import * as fs from 'fs';
+import * as pirates from 'pirates';
+import * as babel from '@babel/core';
+import * as sourceMapSupport from 'source-map-support';
+
+const version = 4;
+const cacheDir = process.env.PWTEST_CACHE_DIR || path.join(os.tmpdir(), 'playwright-transform-cache');
+const sourceMaps: Map<string, string> = new Map();
+
+sourceMapSupport.install({
+  environment: 'node',
+  handleUncaughtExceptions: false,
+  retrieveSourceMap(source) {
+    if (!sourceMaps.has(source))
+      return null;
+    const sourceMapPath = sourceMaps.get(source)!;
+    if (!fs.existsSync(sourceMapPath))
+      return null;
+    return {
+      map: JSON.parse(fs.readFileSync(sourceMapPath, 'utf-8')),
+      url: source
+    };
+  }
+});
+
+function calculateCachePath(content: string, filePath: string): string {
+  const hash = crypto.createHash('sha1').update(content).update(filePath).update(String(version)).digest('hex');
+  const fileName = path.basename(filePath, path.extname(filePath)).replace(/\W/g, '') + '_' + hash;
+  return path.join(cacheDir, hash[0] + hash[1], fileName);
+}
+
+export function installTransform(): () => void {
+  return pirates.addHook((code, filename) => {
+    const cachePath = calculateCachePath(code, filename);
+    const codePath = cachePath + '.js';
+    const sourceMapPath = cachePath + '.map';
+    sourceMaps.set(filename, sourceMapPath);
+    if (fs.existsSync(codePath))
+      return fs.readFileSync(codePath, 'utf8');
+    // We don't use any browserslist data, but babel checks it anyway.
+    // Silence the annoying warning.
+    process.env.BROWSERSLIST_IGNORE_OLD_DATA = 'true';
+    const result = babel.transformFileSync(filename, {
+      babelrc: false,
+      configFile: false,
+      assumptions: {
+        // Without this, babel defines a top level function that
+        // breaks playwright evaluates.
+        setPublicClassFields: true,
+      },
+      presets: [
+        [require.resolve('@babel/preset-typescript'), { onlyRemoveTypeImports: true }],
+      ],
+      plugins: [
+        [require.resolve('@babel/plugin-proposal-class-properties')],
+        [require.resolve('@babel/plugin-proposal-numeric-separator')],
+        [require.resolve('@babel/plugin-proposal-logical-assignment-operators')],
+        [require.resolve('@babel/plugin-proposal-nullish-coalescing-operator')],
+        [require.resolve('@babel/plugin-proposal-optional-chaining')],
+        [require.resolve('@babel/plugin-syntax-json-strings')],
+        [require.resolve('@babel/plugin-syntax-optional-catch-binding')],
+        [require.resolve('@babel/plugin-syntax-async-generators')],
+        [require.resolve('@babel/plugin-syntax-object-rest-spread')],
+        [require.resolve('@babel/plugin-proposal-export-namespace-from')],
+        [require.resolve('@babel/plugin-transform-modules-commonjs')],
+        [require.resolve('@babel/plugin-proposal-dynamic-import')],
+      ],
+      sourceMaps: 'both',
+    } as babel.TransformOptions)!;
+    if (result.code) {
+      fs.mkdirSync(path.dirname(cachePath), {recursive: true});
+      if (result.map)
+        fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8');
+      fs.writeFileSync(codePath, result.code, 'utf8');
+    }
+    return result.code || '';
+  }, {
+    exts: ['.ts']
+  });
+}

src/test/types.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from '../../types/testInternal';

src/test/util.ts

@@ -0,0 +1,195 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import util from 'util';
+import StackUtils from 'stack-utils';
+import type { Location } from './types';
+import type { TestError } from './reporter';
+import { default as minimatch } from 'minimatch';
+
+const TEST_RUNNER_DIRS = [
+  path.join('@playwright', 'test', 'lib'),
+  path.join(__dirname, '..', '..', 'src', 'test'),
+];
+const cwd = process.cwd();
+const stackUtils = new StackUtils({ cwd });
+
+export class DeadlineRunner<T> {
+  private _timer: NodeJS.Timer | undefined;
+  private _done = false;
+  private _fulfill!: (t: { result?: T, timedOut?: boolean }) => void;
+  private _reject!: (error: any) => void;
+
+  readonly result: Promise<{ result?: T, timedOut?: boolean }>;
+
+  constructor(promise: Promise<T>, deadline: number | undefined) {
+    this.result = new Promise((f, r) => {
+      this._fulfill = f;
+      this._reject = r;
+    });
+    promise.then(result => {
+      this._finish({ result });
+    }).catch(e => {
+      this._finish(undefined, e);
+    });
+    this.setDeadline(deadline);
+  }
+
+  private _finish(success?: { result?: T, timedOut?: boolean }, error?: any) {
+    if (this._done)
+      return;
+    this.setDeadline(undefined);
+    if (success)
+      this._fulfill(success);
+    else
+      this._reject(error);
+  }
+
+  setDeadline(deadline: number | undefined) {
+    if (this._timer) {
+      clearTimeout(this._timer);
+      this._timer = undefined;
+    }
+    if (deadline === undefined)
+      return;
+    const timeout = deadline - monotonicTime();
+    if (timeout <= 0)
+      this._finish({ timedOut: true });
+    else
+      this._timer = setTimeout(() => this._finish({ timedOut: true }), timeout);
+  }
+}
+
+export async function raceAgainstDeadline<T>(promise: Promise<T>, deadline: number | undefined): Promise<{ result?: T, timedOut?: boolean }> {
+  return (new DeadlineRunner(promise, deadline)).result;
+}
+
+export function serializeError(error: Error | any): TestError {
+  if (error instanceof Error) {
+    return {
+      message: error.message,
+      stack: error.stack
+    };
+  }
+  return {
+    value: util.inspect(error)
+  };
+}
+
+function callFrames(): string[] {
+  const obj = { stack: '' };
+  Error.captureStackTrace(obj);
+  const frames = obj.stack.split('\n').slice(1);
+  while (frames.length && TEST_RUNNER_DIRS.some(dir => frames[0].includes(dir)))
+    frames.shift();
+  return frames;
+}
+
+export function callLocation(fallbackFile?: string): Location {
+  const frames = callFrames();
+  if (!frames.length)
+    return {file: fallbackFile || '<unknown>', line: 1, column: 1};
+  const location = stackUtils.parseLine(frames[0])!;
+  return {
+    file: path.resolve(cwd, location.file || ''),
+    line: location.line || 0,
+    column: location.column || 0,
+  };
+}
+
+export function errorWithCallLocation(message: string): Error {
+  const frames = callFrames();
+  const error = new Error(message);
+  error.stack = 'Error: ' + message + '\n' + frames.join('\n');
+  return error;
+}
+
+export function monotonicTime(): number {
+  const [seconds, nanoseconds] = process.hrtime();
+  return seconds * 1000 + (nanoseconds / 1000000 | 0);
+}
+
+export function prependErrorMessage(e: Error, message: string) {
+  let stack = e.stack || '';
+  if (stack.includes(e.message))
+    stack = stack.substring(stack.indexOf(e.message) + e.message.length);
+  let m = e.message;
+  if (m.startsWith('Error:'))
+    m = m.substring('Error:'.length);
+  e.message = message + m;
+  e.stack = e.message + stack;
+}
+
+export function isRegExp(e: any): e is RegExp {
+  return e && typeof e === 'object' && (e instanceof RegExp || Object.prototype.toString.call(e) === '[object RegExp]');
+}
+
+export type Matcher = (value: string) => boolean;
+
+export function createMatcher(patterns: string | RegExp | (string | RegExp)[]): Matcher {
+  const reList: RegExp[] = [];
+  const filePatterns: string[] = [];
+  for (const pattern of Array.isArray(patterns) ? patterns : [patterns]) {
+    if (isRegExp(pattern)) {
+      reList.push(pattern);
+    } else {
+      if (!pattern.startsWith('**/') && !pattern.startsWith('**/'))
+        filePatterns.push('**/' + pattern);
+      else
+        filePatterns.push(pattern);
+    }
+  }
+
+  return (value: string) => {
+    for (const re of reList) {
+      re.lastIndex = 0;
+      if (re.test(value))
+        return true;
+    }
+    for (const pattern of filePatterns) {
+      if (minimatch(value, pattern))
+        return true;
+    }
+    return false;
+  };
+}
+
+export function mergeObjects<A extends object, B extends object>(a: A | undefined | void, b: B | undefined | void): A & B {
+  const result = { ...a } as any;
+  if (!Object.is(b, undefined)) {
+    for (const [name, value] of Object.entries(b as B)) {
+      if (!Object.is(value, undefined))
+        result[name] = value;
+    }
+  }
+  return result as any;
+}
+
+export async function wrapInPromise(value: any) {
+  return value;
+}
+
+export function formatLocation(location: Location) {
+  return location.file + ':' + location.line + ':' + location.column;
+}
+
+export function forceRegExp(pattern: string): RegExp {
+  const match = pattern.match(/^\/(.*)\/([gi]*)$/);
+  if (match)
+    return new RegExp(match[1], match[2]);
+  return new RegExp(pattern, 'g');
+}

src/test/worker.ts

@@ -0,0 +1,119 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Console } from 'console';
+import * as util from 'util';
+import { RunPayload, TestOutputPayload, WorkerInitParams } from './ipc';
+import { serializeError } from './util';
+import { WorkerRunner } from './workerRunner';
+
+let closed = false;
+
+sendMessageToParent('ready');
+
+global.console = new Console({
+  stdout: process.stdout,
+  stderr: process.stderr,
+  colorMode: process.env.FORCE_COLOR === '1',
+});
+
+process.stdout.write = (chunk: string | Buffer) => {
+  const outPayload: TestOutputPayload = {
+    testId: workerRunner?._currentTest?.testId,
+    ...chunkToParams(chunk)
+  };
+  sendMessageToParent('stdOut', outPayload);
+  return true;
+};
+
+if (!process.env.PW_RUNNER_DEBUG) {
+  process.stderr.write = (chunk: string | Buffer) => {
+    const outPayload: TestOutputPayload = {
+      testId: workerRunner?._currentTest?.testId,
+      ...chunkToParams(chunk)
+    };
+    sendMessageToParent('stdErr', outPayload);
+    return true;
+  };
+}
+
+process.on('disconnect', gracefullyCloseAndExit);
+process.on('SIGINT',() => {});
+process.on('SIGTERM',() => {});
+
+let workerRunner: WorkerRunner;
+
+process.on('unhandledRejection', (reason, promise) => {
+  if (workerRunner)
+    workerRunner.unhandledError(reason);
+});
+
+process.on('uncaughtException', error => {
+  if (workerRunner)
+    workerRunner.unhandledError(error);
+});
+
+process.on('message', async message => {
+  if (message.method === 'init') {
+    const initParams = message.params as WorkerInitParams;
+    workerRunner = new WorkerRunner(initParams);
+    for (const event of ['testBegin', 'testEnd', 'done'])
+      workerRunner.on(event, sendMessageToParent.bind(null, event));
+    return;
+  }
+  if (message.method === 'stop') {
+    await gracefullyCloseAndExit();
+    return;
+  }
+  if (message.method === 'run') {
+    const runPayload = message.params as RunPayload;
+    await workerRunner!.run(runPayload);
+  }
+});
+
+async function gracefullyCloseAndExit() {
+  if (closed)
+    return;
+  closed = true;
+  // Force exit after 30 seconds.
+  setTimeout(() => process.exit(0), 30000);
+  // Meanwhile, try to gracefully shutdown.
+  try {
+    if (workerRunner) {
+      workerRunner.stop();
+      await workerRunner.cleanup();
+    }
+  } catch (e) {
+    process.send!({ method: 'teardownError', params: { error: serializeError(e) } });
+  }
+  process.exit(0);
+}
+
+function sendMessageToParent(method: string, params = {}) {
+  try {
+    process.send!({ method, params });
+  } catch (e) {
+    // Can throw when closing.
+  }
+}
+
+function chunkToParams(chunk: Buffer | string):  { text?: string, buffer?: string } {
+  if (chunk instanceof Buffer)
+    return { buffer: chunk.toString('base64') };
+  if (typeof chunk !== 'string')
+    return { text: util.inspect(chunk) };
+  return { text: chunk };
+}

src/test/workerRunner.ts

@@ -0,0 +1,453 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import rimraf from 'rimraf';
+import util from 'util';
+import { EventEmitter } from 'events';
+import { monotonicTime, DeadlineRunner, raceAgainstDeadline, serializeError } from './util';
+import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams } from './ipc';
+import { setCurrentTestInfo } from './globals';
+import { Loader } from './loader';
+import { Spec, Suite, Test } from './test';
+import { TestInfo, WorkerInfo } from './types';
+import { ProjectImpl } from './project';
+import { FixtureRunner } from './fixtures';
+
+const removeFolderAsync = util.promisify(rimraf);
+
+export class WorkerRunner extends EventEmitter {
+  private _params: WorkerInitParams;
+  private _loader!: Loader;
+  private _project!: ProjectImpl;
+  private _workerInfo!: WorkerInfo;
+  private _projectNamePathSegment = '';
+  private _uniqueProjectNamePathSegment = '';
+  private _fixtureRunner: FixtureRunner;
+
+  private _failedTestId: string | undefined;
+  private _fatalError: any | undefined;
+  private _entries = new Map<string, TestEntry>();
+  private _remaining = new Map<string, TestEntry>();
+  private _isStopped: any;
+  _currentTest: { testId: string, testInfo: TestInfo } | null = null;
+
+  constructor(params: WorkerInitParams) {
+    super();
+    this._params = params;
+    this._fixtureRunner = new FixtureRunner();
+  }
+
+  stop() {
+    this._isStopped = true;
+    this._setCurrentTest(null);
+  }
+
+  async cleanup() {
+    // TODO: separate timeout for teardown?
+    const result = await raceAgainstDeadline((async () => {
+      await this._fixtureRunner.teardownScope('test');
+      await this._fixtureRunner.teardownScope('worker');
+    })(), this._deadline());
+    if (result.timedOut)
+      throw new Error(`Timeout of ${this._project.config.timeout}ms exceeded while shutting down environment`);
+  }
+
+  unhandledError(error: Error | any) {
+    if (this._isStopped)
+      return;
+    if (this._currentTest) {
+      this._currentTest.testInfo.status = 'failed';
+      this._currentTest.testInfo.error = serializeError(error);
+      this._failedTestId = this._currentTest.testId;
+      this.emit('testEnd', buildTestEndPayload(this._currentTest.testId, this._currentTest.testInfo));
+    } else {
+      // No current test - fatal error.
+      this._fatalError = serializeError(error);
+    }
+    this._reportDoneAndStop();
+  }
+
+  private _deadline() {
+    return this._project.config.timeout ? monotonicTime() + this._project.config.timeout : undefined;
+  }
+
+  private _loadIfNeeded() {
+    if (this._loader)
+      return;
+
+    this._loader = Loader.deserialize(this._params.loader);
+    this._project = this._loader.projects()[this._params.projectIndex];
+
+    this._projectNamePathSegment = sanitizeForFilePath(this._project.config.name);
+
+    const sameName = this._loader.projects().filter(project => project.config.name === this._project.config.name);
+    if (sameName.length > 1)
+      this._uniqueProjectNamePathSegment = this._project.config.name + (sameName.indexOf(this._project) + 1);
+    else
+      this._uniqueProjectNamePathSegment = this._project.config.name;
+    this._uniqueProjectNamePathSegment = sanitizeForFilePath(this._uniqueProjectNamePathSegment);
+
+    this._workerInfo = {
+      workerIndex: this._params.workerIndex,
+      project: this._project.config,
+      config: this._loader.fullConfig(),
+    };
+  }
+
+  async run(runPayload: RunPayload) {
+    this._entries = new Map(runPayload.entries.map(e => [ e.testId, e ]));
+    this._remaining = new Map(runPayload.entries.map(e => [ e.testId, e ]));
+
+    this._loadIfNeeded();
+    const fileSuite = this._loader.loadTestFile(runPayload.file);
+    let anySpec: Spec | undefined;
+    fileSuite.findSpec(spec => {
+      const test = this._project.generateTests(spec, this._params.repeatEachIndex)[0];
+      if (this._entries.has(test._id))
+        anySpec = spec;
+    });
+    if (!anySpec) {
+      this._reportDone();
+      return;
+    }
+
+    this._fixtureRunner.setPool(this._project.buildPool(anySpec));
+    await this._runSuite(fileSuite);
+    if (this._isStopped)
+      return;
+
+    this._reportDone();
+  }
+
+  private async _runSuite(suite: Suite) {
+    if (this._isStopped)
+      return;
+    const skipHooks = !this._hasTestsToRun(suite);
+    for (const hook of suite._hooks) {
+      if (hook.type !== 'beforeAll' || skipHooks)
+        continue;
+      if (this._isStopped)
+        return;
+      // TODO: separate timeout for beforeAll?
+      const result = await raceAgainstDeadline(this._fixtureRunner.resolveParametersAndRunHookOrTest(hook.fn, 'worker', this._workerInfo), this._deadline());
+      if (result.timedOut) {
+        this._fatalError = serializeError(new Error(`Timeout of ${this._project.config.timeout}ms exceeded while running beforeAll hook`));
+        this._reportDoneAndStop();
+      }
+    }
+    for (const entry of suite._entries) {
+      if (entry instanceof Suite)
+        await this._runSuite(entry);
+      else
+        await this._runSpec(entry);
+    }
+    for (const hook of suite._hooks) {
+      if (hook.type !== 'afterAll' || skipHooks)
+        continue;
+      if (this._isStopped)
+        return;
+      // TODO: separate timeout for afterAll?
+      const result = await raceAgainstDeadline(this._fixtureRunner.resolveParametersAndRunHookOrTest(hook.fn, 'worker', this._workerInfo), this._deadline());
+      if (result.timedOut) {
+        this._fatalError = serializeError(new Error(`Timeout of ${this._project.config.timeout}ms exceeded while running afterAll hook`));
+        this._reportDoneAndStop();
+      }
+    }
+  }
+
+  private async _runSpec(spec: Spec) {
+    if (this._isStopped)
+      return;
+    const test = spec.tests[0];
+    const entry = this._entries.get(test._id);
+    if (!entry)
+      return;
+    this._remaining.delete(test._id);
+
+    const startTime = monotonicTime();
+    let deadlineRunner: DeadlineRunner<any> | undefined;
+    const testId = test._id;
+
+    const baseOutputDir = (() => {
+      const relativeTestFilePath = path.relative(this._project.config.testDir, spec.file.replace(/\.(spec|test)\.(js|ts)/, ''));
+      const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-');
+      let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(spec.title);
+      if (this._uniqueProjectNamePathSegment)
+        testOutputDir += '-' + this._uniqueProjectNamePathSegment;
+      if (entry.retry)
+        testOutputDir += '-retry' + entry.retry;
+      if (this._params.repeatEachIndex)
+        testOutputDir += '-repeat' + this._params.repeatEachIndex;
+      return path.join(this._project.config.outputDir, testOutputDir);
+    })();
+
+    const testInfo: TestInfo = {
+      ...this._workerInfo,
+      title: spec.title,
+      file: spec.file,
+      line: spec.line,
+      column: spec.column,
+      fn: spec.fn,
+      repeatEachIndex: this._params.repeatEachIndex,
+      retry: entry.retry,
+      expectedStatus: 'passed',
+      annotations: [],
+      duration: 0,
+      status: 'passed',
+      stdout: [],
+      stderr: [],
+      timeout: this._project.config.timeout,
+      snapshotSuffix: '',
+      outputDir: baseOutputDir,
+      outputPath: (...pathSegments: string[]): string => {
+        fs.mkdirSync(baseOutputDir, { recursive: true });
+        return path.join(baseOutputDir, ...pathSegments);
+      },
+      snapshotPath: (snapshotName: string): string => {
+        let suffix = '';
+        if (this._projectNamePathSegment)
+          suffix += '-' + this._projectNamePathSegment;
+        if (testInfo.snapshotSuffix)
+          suffix += '-' + testInfo.snapshotSuffix;
+        if (suffix) {
+          const ext = path.extname(snapshotName);
+          if (ext)
+            snapshotName = snapshotName.substring(0, snapshotName.length - ext.length) + suffix + ext;
+          else
+            snapshotName += suffix;
+        }
+        return path.join(spec.file + '-snapshots', snapshotName);
+      },
+      skip: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'skip', args),
+      fixme: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fixme', args),
+      fail: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fail', args),
+      slow: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'slow', args),
+      setTimeout: (timeout: number) => {
+        testInfo.timeout = timeout;
+        if (deadlineRunner)
+          deadlineRunner.setDeadline(deadline());
+      },
+    };
+    this._setCurrentTest({ testInfo, testId });
+    const deadline = () => {
+      return testInfo.timeout ? startTime + testInfo.timeout : undefined;
+    };
+
+    this.emit('testBegin', buildTestBeginPayload(testId, testInfo));
+
+    if (testInfo.expectedStatus === 'skipped') {
+      testInfo.status = 'skipped';
+      this.emit('testEnd', buildTestEndPayload(testId, testInfo));
+      return;
+    }
+
+    // Update the fixture pool - it may differ between tests, but only in test-scoped fixtures.
+    this._fixtureRunner.setPool(this._project.buildPool(spec));
+
+    deadlineRunner = new DeadlineRunner(this._runTestWithBeforeHooks(test, testInfo), deadline());
+    const result = await deadlineRunner.result;
+    // Do not overwrite test failure upon hook timeout.
+    if (result.timedOut && testInfo.status === 'passed')
+      testInfo.status = 'timedOut';
+    if (this._isStopped)
+      return;
+
+    if (!result.timedOut) {
+      deadlineRunner = new DeadlineRunner(this._runAfterHooks(test, testInfo), deadline());
+      deadlineRunner.setDeadline(deadline());
+      const hooksResult = await deadlineRunner.result;
+      // Do not overwrite test failure upon hook timeout.
+      if (hooksResult.timedOut && testInfo.status === 'passed')
+        testInfo.status = 'timedOut';
+    } else {
+      // A timed-out test gets a full additional timeout to run after hooks.
+      const newDeadline = this._deadline();
+      deadlineRunner = new DeadlineRunner(this._runAfterHooks(test, testInfo), newDeadline);
+      await deadlineRunner.result;
+    }
+
+    if (this._isStopped)
+      return;
+
+    testInfo.duration = monotonicTime() - startTime;
+    this.emit('testEnd', buildTestEndPayload(testId, testInfo));
+
+    const isFailure = testInfo.status === 'timedOut' || (testInfo.status === 'failed' && testInfo.expectedStatus !== 'failed');
+    const preserveOutput = this._loader.fullConfig().preserveOutput === 'always' ||
+      (this._loader.fullConfig().preserveOutput === 'failures-only' && isFailure);
+    if (!preserveOutput)
+      await removeFolderAsync(testInfo.outputDir).catch(e => {});
+
+    if (testInfo.status !== 'passed') {
+      this._failedTestId = testId;
+      this._reportDoneAndStop();
+    }
+    this._setCurrentTest(null);
+  }
+
+  private _setCurrentTest(currentTest: { testId: string, testInfo: TestInfo} | null) {
+    this._currentTest = currentTest;
+    setCurrentTestInfo(currentTest ? currentTest.testInfo : null);
+  }
+
+  private async _runTestWithBeforeHooks(test: Test, testInfo: TestInfo) {
+    try {
+      await this._runHooks(test.spec.parent!, 'beforeEach', testInfo);
+    } catch (error) {
+      if (error instanceof SkipError) {
+        if (testInfo.status === 'passed')
+          testInfo.status = 'skipped';
+      } else {
+        testInfo.status = 'failed';
+        testInfo.error = serializeError(error);
+      }
+      // Continue running afterEach hooks even after the failure.
+    }
+
+    // Do not run the test when beforeEach hook fails.
+    if (this._isStopped || testInfo.status === 'failed' || testInfo.status === 'skipped')
+      return;
+
+    try {
+      await this._fixtureRunner.resolveParametersAndRunHookOrTest(test.spec.fn, 'test', testInfo);
+    } catch (error) {
+      if (error instanceof SkipError) {
+        if (testInfo.status === 'passed')
+          testInfo.status = 'skipped';
+      } else {
+        // We might fail after the timeout, e.g. due to fixture teardown.
+        // Do not overwrite the timeout status with this error.
+        if (testInfo.status === 'passed') {
+          testInfo.status = 'failed';
+          testInfo.error = serializeError(error);
+        }
+      }
+    }
+  }
+
+  private async _runAfterHooks(test: Test, testInfo: TestInfo) {
+    try {
+      await this._runHooks(test.spec.parent!, 'afterEach', testInfo);
+    } catch (error) {
+      // Do not overwrite test failure error.
+      if (!(error instanceof SkipError) && testInfo.status === 'passed') {
+        testInfo.status = 'failed';
+        testInfo.error = serializeError(error);
+        // Continue running even after the failure.
+      }
+    }
+    try {
+      await this._fixtureRunner.teardownScope('test');
+    } catch (error) {
+      // Do not overwrite test failure error.
+      if (testInfo.status === 'passed') {
+        testInfo.status = 'failed';
+        testInfo.error = serializeError(error);
+      }
+    }
+  }
+
+  private async _runHooks(suite: Suite, type: 'beforeEach' | 'afterEach', testInfo: TestInfo) {
+    if (this._isStopped)
+      return;
+    const all = [];
+    for (let s: Suite | undefined = suite; s; s = s.parent) {
+      const funcs = s._hooks.filter(e => e.type === type).map(e => e.fn);
+      all.push(...funcs.reverse());
+    }
+    if (type === 'beforeEach')
+      all.reverse();
+    let error: Error | undefined;
+    for (const hook of all) {
+      try {
+        await this._fixtureRunner.resolveParametersAndRunHookOrTest(hook, 'test', testInfo);
+      } catch (e) {
+        // Always run all the hooks, and capture the first error.
+        error = error || e;
+      }
+    }
+    if (error)
+      throw error;
+  }
+
+  private _reportDone() {
+    const donePayload: DonePayload = {
+      failedTestId: this._failedTestId,
+      fatalError: this._fatalError,
+      remaining: [...this._remaining.values()],
+    };
+    this.emit('done', donePayload);
+  }
+
+  private _reportDoneAndStop() {
+    if (this._isStopped)
+      return;
+    this._reportDone();
+    this.stop();
+  }
+
+  private _hasTestsToRun(suite: Suite): boolean {
+    return suite.findSpec(spec => {
+      const entry = this._entries.get(spec.tests[0]._id);
+      return !!entry;
+    });
+  }
+}
+
+function buildTestBeginPayload(testId: string, testInfo: TestInfo): TestBeginPayload {
+  return {
+    testId,
+    workerIndex: testInfo.workerIndex
+  };
+}
+
+function buildTestEndPayload(testId: string, testInfo: TestInfo): TestEndPayload {
+  return {
+    testId,
+    duration: testInfo.duration,
+    status: testInfo.status!,
+    error: testInfo.error,
+    expectedStatus: testInfo.expectedStatus,
+    annotations: testInfo.annotations,
+    timeout: testInfo.timeout,
+  };
+}
+
+function modifier(testInfo: TestInfo, type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) {
+  if (modifierArgs.length >= 1 && !modifierArgs[0])
+    return;
+
+  const description = modifierArgs[1];
+  testInfo.annotations.push({ type, description });
+  if (type === 'slow') {
+    testInfo.setTimeout(testInfo.timeout * 3);
+  } else if (type === 'skip' || type === 'fixme') {
+    testInfo.expectedStatus = 'skipped';
+    throw new SkipError('Test is skipped: ' + (description || ''));
+  } else if (type === 'fail') {
+    if (testInfo.expectedStatus !== 'skipped')
+      testInfo.expectedStatus = 'failed';
+  }
+}
+
+class SkipError extends Error {
+}
+
+function sanitizeForFilePath(s: string) {
+  return s.replace(/[^\w\d]+/g, '-');
+}

tests/android/androidTest.ts

@@ -16,15 +16,15 @@
 
 import type { AndroidDevice, BrowserContext } from '../../index';
 import { CommonWorkerFixtures, baseTest } from '../config/baseTest';
-import * as folio from 'folio';
+import type { Fixtures } from '../config/test-runner';
 import { PageTestFixtures } from '../page/pageTest';
-export { expect } from 'folio';
+export { expect } from '../config/test-runner';
 
 type AndroidWorkerFixtures = {
   androidDevice: AndroidDevice;
 };
 
-export const androidFixtures: folio.Fixtures<PageTestFixtures, AndroidWorkerFixtures & { androidContext: BrowserContext }, {}, CommonWorkerFixtures> = {
+export const androidFixtures: Fixtures<PageTestFixtures, AndroidWorkerFixtures & { androidContext: BrowserContext }, {}, CommonWorkerFixtures> = {
   androidDevice: [ async ({ playwright }, run) => {
     const device = (await playwright._android.devices())[0];
     await device.shell('am force-stop org.chromium.webview_shell');

tests/config/android.config.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import * as folio from 'folio';
+import type { Config } from '../config/test-runner';
 import * as path from 'path';
 import { test as pageTest } from '../page/pageTest';
 import { androidFixtures } from '../android/androidTest';
@@ -23,7 +23,7 @@ import { CommonOptions } from './baseTest';
 
 const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
-const config: folio.Config<CommonOptions & PlaywrightOptions> = {
+const config: Config<CommonOptions & PlaywrightOptions> = {
   testDir,
   outputDir,
   timeout: 120000,

tests/config/baseTest.ts

@@ -15,7 +15,7 @@
  */
 
 import { TestServer } from '../../utils/testserver';
-import * as folio from 'folio';
+import { Fixtures, TestType } from './test-runner';
 import * as path from 'path';
 import * as fs from 'fs';
 import socks from 'socksv5';
@@ -101,7 +101,7 @@ class DefaultMode {
   }
 }
 
-const baseFixtures: folio.Fixtures<{}, BaseOptions & BaseFixtures> = {
+const baseFixtures: Fixtures<{}, BaseOptions & BaseFixtures> = {
   mode: [ 'default', { scope: 'worker' } ],
   browserName: [ 'chromium' , { scope: 'worker' } ],
   channel: [ undefined, { scope: 'worker' } ],
@@ -136,7 +136,7 @@ type ServerFixtures = {
 };
 
 type ServersInternal = ServerFixtures & { socksServer: any };
-const serverFixtures: folio.Fixtures<ServerFixtures, ServerOptions & { __servers: ServersInternal }> = {
+const serverFixtures: Fixtures<ServerFixtures, ServerOptions & { __servers: ServersInternal }> = {
   loopback: [ undefined, { scope: 'worker' } ],
   __servers: [ async ({ loopback }, run, workerInfo) => {
     const assetsPath = path.join(__dirname, '..', 'assets');
@@ -208,7 +208,7 @@ type CoverageOptions = {
   coverageName?: string;
 };
 
-const coverageFixtures: folio.Fixtures<{}, CoverageOptions & { __collectCoverage: void }> = {
+const coverageFixtures: Fixtures<{}, CoverageOptions & { __collectCoverage: void }> = {
   coverageName: [ undefined, { scope: 'worker' } ],
 
   __collectCoverage: [ async ({ coverageName }, run, workerInfo) => {
@@ -230,4 +230,5 @@ const coverageFixtures: folio.Fixtures<{}, CoverageOptions & { __collectCoverage
 export type CommonOptions = BaseOptions & ServerOptions & CoverageOptions;
 export type CommonWorkerFixtures = CommonOptions & BaseFixtures;
 
-export const baseTest = folio.test.extend<{}, CoverageOptions>(coverageFixtures).extend<ServerFixtures>(serverFixtures).extend<{}, BaseOptions & BaseFixtures>(baseFixtures);
+const __baseTest = require('./test-runner').__baseTest as TestType<{}, {}>;
+export const baseTest = __baseTest.extend<{}, CoverageOptions>(coverageFixtures).extend<ServerFixtures>(serverFixtures).extend<{}, BaseOptions & BaseFixtures>(baseFixtures);

tests/config/browserTest.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import * as folio from 'folio';
+import type { Fixtures } from './test-runner';
 import type { Browser, BrowserContext, BrowserContextOptions, BrowserType, LaunchOptions, Page } from '../../index';
 import { removeFolders } from '../../lib/utils/utils';
 import * as path from 'path';
@@ -49,7 +49,7 @@ type PlaywrightTestFixtures = {
 };
 export type PlaywrightOptions = PlaywrightWorkerOptions & PlaywrightTestOptions;
 
-export const playwrightFixtures: folio.Fixtures<PlaywrightTestOptions & PlaywrightTestFixtures, PlaywrightWorkerOptions & PlaywrightWorkerFixtures, {}, CommonWorkerFixtures> = {
+export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTestFixtures, PlaywrightWorkerOptions & PlaywrightWorkerFixtures, {}, CommonWorkerFixtures> = {
   tracesDir: [ undefined, { scope: 'worker' } ],
   executablePath: [ undefined, { scope: 'worker' } ],
   proxy: [ undefined, { scope: 'worker' } ],
@@ -159,4 +159,4 @@ export const playwrightTest = test;
 export const browserTest = test;
 export const contextTest = test;
 
-export { expect } from 'folio';
+export { expect } from './test-runner';

tests/config/default.config.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import * as folio from 'folio';
+import type { Config } from './test-runner';
 import * as path from 'path';
 import { PlaywrightOptions, playwrightFixtures } from './browserTest';
 import { test as pageTest } from '../page/pageTest';
@@ -45,7 +45,7 @@ const video = !!process.env.PWTEST_VIDEO;
 
 const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
-const config: folio.Config<CommonOptions & PlaywrightOptions> = {
+const config: Config<CommonOptions & PlaywrightOptions> = {
   testDir,
   outputDir,
   timeout: video || process.env.PWTRACE ? 60000 : 30000,
@@ -66,7 +66,7 @@ for (const browserName of browserNames) {
   if (executablePath && !process.env.TEST_WORKER_INDEX)
     console.error(`Using executable at ${executablePath}`);
   const testIgnore: RegExp[] = browserNames.filter(b => b !== browserName).map(b => new RegExp(b));
-  testIgnore.push(/android/, /electron/);
+  testIgnore.push(/android/, /electron/, /playwrigh-test/);
   config.projects.push({
     name: browserName,
     testDir,

tests/config/electron.config.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import * as folio from 'folio';
+import type { Config } from './test-runner';
 import * as path from 'path';
 import { electronFixtures } from '../electron/electronTest';
 import { test as pageTest } from '../page/pageTest';
@@ -23,7 +23,7 @@ import { CommonOptions } from './baseTest';
 
 const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
-const config: folio.Config<CommonOptions & PlaywrightOptions> = {
+const config: Config<CommonOptions & PlaywrightOptions> = {
   testDir,
   outputDir,
   timeout: 30000,

tests/config/test-runner/.gitignore

@@ -0,0 +1 @@
+/node_modules/

tests/config/test-runner/README.md

@@ -0,0 +1,3 @@
+This directory holds a stable test runner:
+- It is possible to test broken test runner with a stable working one.
+- Dependencies between ToT and stable test runner do not clash.

tests/config/test-runner/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './node_modules/@playwright/test';

tests/config/test-runner/index.js

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module.exports = require('./node_modules/@playwright/test');

tests/config/test-runner/package.json

@@ -0,0 +1,6 @@
+{
+  "private": true,
+  "dependencies": {
+    "@playwright/test": "=1.12.0-next-1622928816000"
+  }
+}

tests/config/utils.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { expect } from 'folio';
+import { expect } from './test-runner';
 import type { Frame, Page } from '../../index';
 
 export async function attachFrame(page: Page, frameId: string, url: string): Promise<Frame> {

tests/electron/electronTest.ts

@@ -16,10 +16,10 @@
 
 import { baseTest, CommonWorkerFixtures } from '../config/baseTest';
 import { ElectronApplication, Page } from '../../index';
-import * as folio from 'folio';
+import type { Fixtures } from '../config/test-runner';
 import * as path from 'path';
 import { PageTestFixtures } from '../page/pageTest';
-export { expect } from 'folio';
+export { expect } from '../config/test-runner';
 
 type ElectronTestFixtures = PageTestFixtures & {
   electronApp: ElectronApplication;
@@ -27,7 +27,7 @@ type ElectronTestFixtures = PageTestFixtures & {
 };
 
 const electronVersion = require('electron/package.json').version;
-export const electronFixtures: folio.Fixtures<ElectronTestFixtures, {}, {}, CommonWorkerFixtures> = {
+export const electronFixtures: Fixtures<ElectronTestFixtures, {}, {}, CommonWorkerFixtures> = {
   browserVersion: electronVersion,
   browserMajorVersion: Number(electronVersion.split('.')[0]),
   isAndroid: false,

tests/inspector/inspectorTest.ts

@@ -20,7 +20,7 @@ import * as path from 'path';
 import type { Source } from '../../src/server/supplements/recorder/recorderTypes';
 import { ChildProcess, spawn } from 'child_process';
 import { chromium } from '../../index';
-export { expect } from 'folio';
+export { expect } from '../config/test-runner';
 
 type CLITestArgs = {
   recorderPageGetter: () => Promise<Page>;

tests/page/pageTest.ts

@@ -16,7 +16,7 @@
 
 import { baseTest } from '../config/baseTest';
 import type { Page } from '../../index';
-export { expect } from 'folio';
+export { expect } from '../config/test-runner';
 
 // Page test does not guarantee an isolated context, just a new page (because Android).
 export type PageTestFixtures = {

tests/playwright-test/access-data.spec.ts

@@ -0,0 +1,84 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should access error in fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'test-error-visible-in-env.spec.ts': `
+      const test = folio.test.extend({
+        foo: [async ({}, run, testInfo) => {
+          await run();
+          console.log('ERROR[[[' + JSON.stringify(testInfo.error, undefined, 2) + ']]]');
+        }, { auto: true }],
+      });
+      test('ensure env handles test error', async ({}) => {
+        expect(true).toBe(false);
+      });
+    `
+  }, {});
+  expect(result.exitCode).toBe(1);
+  const start = result.output.indexOf('ERROR[[[') + 8;
+  const end = result.output.indexOf(']]]');
+  const data = JSON.parse(result.output.substring(start, end));
+  expect(data.message).toContain('Object.is equality');
+});
+
+test('should access annotations in fixture', async ({ runInlineTest }) => {
+  const { exitCode, report } = await runInlineTest({
+    'test-data-visible-in-env.spec.ts': `
+      const test = folio.test.extend({
+        foo: [async ({}, run, testInfo) => {
+          await run();
+          testInfo.annotations.push({ type: 'myname', description: 'hello' });
+        }, { auto: true }],
+      });
+      test('ensure env can set data', async ({}, testInfo) => {
+        test.slow(true, 'just slow');
+
+        console.log('console.log');
+        console.error('console.error');
+        expect(testInfo.config.rootDir).toBeTruthy();
+        expect(testInfo.file).toContain('test-data-visible-in-env');
+      });
+    `
+  });
+  expect(exitCode).toBe(0);
+  const test = report.suites[0].specs[0].tests[0];
+  expect(test.annotations).toEqual([ { type: 'slow', description: 'just slow' }, { type: 'myname', description: 'hello' } ]);
+  expect(test.results[0].stdout).toEqual([{ text: 'console.log\n' }]);
+  expect(test.results[0].stderr).toEqual([{ text: 'console.error\n' }]);
+});
+
+test('should report projectName in result', async ({ runInlineTest }) => {
+  const { exitCode, report } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'foo' },
+          {},
+        ],
+      };
+    `,
+    'test-data-visible-in-env.spec.ts': `
+      folio.test('some test', async ({}, testInfo) => {
+      });
+    `
+  });
+  expect(report.suites[0].specs[0].tests[0].projectName).toBe('foo');
+  expect(report.suites[0].specs[0].tests[1].projectName).toBe('');
+  expect(exitCode).toBe(0);
+});

tests/playwright-test/base-reporter.spec.ts

@@ -0,0 +1,95 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+
+test('handle long test names', async ({ runInlineTest }) => {
+  const title = 'title'.repeat(30);
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('${title}', async ({}) => {
+        expect(1).toBe(0);
+      });
+    `,
+  });
+  expect(stripAscii(result.output)).toContain('expect(1).toBe');
+  expect(result.exitCode).toBe(1);
+});
+
+test('print the error name', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+    const { test } = folio;
+    test('foobar', async ({}) => {
+      const error = new Error('my-message');
+      error.name = 'FooBarError';
+      throw error;
+    });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('FooBarError: my-message');
+});
+
+test('print should print the error name without a message', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+    const { test } = folio;
+    test('foobar', async ({}) => {
+      const error = new Error();
+      error.name = 'FooBarError';
+      throw error;
+    });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('FooBarError');
+});
+
+test('print an error in a codeframe', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'my-lib.ts': `
+    const foobar = () => {
+      const error = new Error('my-message');
+      error.name = 'FooBarError';
+      throw error;
+    }
+    export default () => {
+      foobar();
+    }
+    `,
+    'a.spec.ts': `
+    const { test } = folio;
+    import myLib from './my-lib';
+    test('foobar', async ({}) => {
+      const error = new Error('my-message');
+      error.name = 'FooBarError';
+      throw error;
+    });
+    `
+  }, {}, {
+    FORCE_COLOR: '0',
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('FooBarError: my-message');
+  expect(result.output).toContain('test(\'foobar\', async');
+  expect(result.output).toContain('throw error;');
+  expect(result.output).toContain('import myLib from \'./my-lib\';');
+});

tests/playwright-test/basic.spec.ts

@@ -0,0 +1,253 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+import * as path from 'path';
+
+test('should fail', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'one-failure.spec.ts': `
+      const { test } = folio;
+      test('fails', () => {
+        expect(1 + 1).toBe(7);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('1) one-failure.spec.ts:6');
+});
+
+test('should timeout', async ({ runInlineTest }) => {
+  const { exitCode, passed, failed, output } = await runInlineTest({
+    'one-timeout.spec.js': `
+      const { test } = folio;
+      test('timeout', async () => {
+        await new Promise(f => setTimeout(f, 10000));
+      });
+    `
+  }, { timeout: 100 });
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(0);
+  expect(failed).toBe(1);
+  expect(output).toContain('Timeout of 100ms exceeded.');
+});
+
+test('should succeed', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'one-success.spec.js': `
+      const { test } = folio;
+      test('succeeds', () => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(0);
+});
+
+test('should report suite errors', async ({ runInlineTest }) => {
+  const { exitCode, failed, output } = await runInlineTest({
+    'suite-error.spec.js': `
+      if (new Error().stack.includes('workerRunner'))
+        throw new Error('Suite error');
+
+      const { test } = folio;
+      test('passes',() => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  });
+  expect(exitCode).toBe(1);
+  expect(failed).toBe(1);
+  expect(output).toContain('Suite error');
+});
+
+test('should respect nested skip', async ({ runInlineTest }) => {
+  const { exitCode, passed, failed, skipped } = await runInlineTest({
+    'nested-skip.spec.js': `
+      const { test } = folio;
+      test.describe('skipped', () => {
+        test.skip();
+        test('succeeds',() => {
+          expect(1 + 1).toBe(2);
+        });
+      });
+    `
+  });
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(0);
+  expect(failed).toBe(0);
+  expect(skipped).toBe(1);
+});
+
+test('should respect excluded tests', async ({ runInlineTest }) => {
+  const { exitCode, passed } = await runInlineTest({
+    'excluded.spec.ts': `
+      const { test } = folio;
+      test('included test', () => {
+        expect(1 + 1).toBe(2);
+      });
+
+      test('excluded test', () => {
+        test.skip();
+        expect(1 + 1).toBe(3);
+      });
+
+      test('excluded test', () => {
+        test.skip();
+        expect(1 + 1).toBe(3);
+      });
+
+      test.describe('included describe', () => {
+        test('included describe test', () => {
+          expect(1 + 1).toBe(2);
+        });
+      });
+
+      test.describe('excluded describe', () => {
+        test.skip();
+        test('excluded describe test', () => {
+          expect(1 + 1).toBe(3);
+        });
+      });
+    `,
+  });
+  expect(passed).toBe(2);
+  expect(exitCode).toBe(0);
+});
+
+test('should respect focused tests', async ({ runInlineTest }) => {
+  const { exitCode, passed } = await runInlineTest({
+    'focused.spec.ts': `
+      const { test } = folio;
+      test('included test', () => {
+        expect(1 + 1).toBe(3);
+      });
+
+      test.only('focused test', () => {
+        expect(1 + 1).toBe(2);
+      });
+
+      test.only('focused only test', () => {
+        expect(1 + 1).toBe(2);
+      });
+
+      test.describe.only('focused describe', () => {
+        test('describe test', () => {
+          expect(1 + 1).toBe(2);
+        });
+      });
+
+      test.describe('non-focused describe', () => {
+        test('describe test', () => {
+          expect(1 + 1).toBe(3);
+        });
+      });
+
+      test.describe.only('focused describe', () => {
+        test('test1', () => {
+          expect(1 + 1).toBe(2);
+        });
+        test.only('test2', () => {
+          expect(1 + 1).toBe(2);
+        });
+        test('test3', () => {
+          expect(1 + 1).toBe(2);
+        });
+        test.only('test4', () => {
+          expect(1 + 1).toBe(2);
+        });
+      });
+    `
+  });
+  expect(passed).toBe(5);
+  expect(exitCode).toBe(0);
+});
+
+test('skip should take priority over fail', async ({ runInlineTest }) => {
+  const { passed, skipped, failed } = await runInlineTest({
+    'test.spec.ts': `
+      const { test } = folio;
+      test.describe('failing suite', () => {
+        test.fail();
+
+        test('skipped', () => {
+          test.skip();
+          expect(1 + 1).toBe(3);
+        });
+
+        test('passing', () => {
+          expect(1 + 1).toBe(3);
+        });
+        test('passing2', () => {
+          expect(1 + 1).toBe(3);
+        });
+
+        test('failing', () => {
+          expect(1 + 1).toBe(2);
+        });
+      });
+    `
+  });
+  expect(passed).toBe(2);
+  expect(skipped).toBe(1);
+  expect(failed).toBe(1);
+});
+
+test('should focus test from one runTests', async ({ runInlineTest }) => {
+  const { exitCode, passed, skipped, failed } = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = { projects: [
+        { testDir: path.join(__dirname, 'a') },
+        { testDir: path.join(__dirname, 'b') },
+      ] };
+    `,
+    'a/afile.spec.ts': `
+      const { test } = folio;
+      test('just a test', () => {
+        expect(1 + 1).toBe(3);
+      });
+    `,
+    'b/bfile.spec.ts': `
+      const { test } = folio;
+      test.only('focused test', () => {
+        expect(1 + 1).toBe(2);
+      });
+    `,
+  }, { reporter: 'list,json' });
+  expect(passed).toBe(1);
+  expect(failed).toBe(0);
+  expect(skipped).toBe(0);
+  expect(exitCode).toBe(0);
+});
+
+test('should work with default export', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'file.spec.ts': `
+      import t from ${JSON.stringify(path.join(__dirname, 'playwright-test-internal'))};
+      t('passed', () => {
+        t.expect(1 + 1).toBe(2);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(0);
+});

tests/playwright-test/config.spec.ts

@@ -0,0 +1,401 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { test, expect } from './playwright-test-fixtures';
+
+test('should be able to define config', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { timeout: 12345 };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', async ({}, testInfo) => {
+        expect(testInfo.timeout).toBe(12345);
+      });
+    `
+  });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should prioritize project timeout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { timeout: 500, projects: [{ timeout: 10000}, {}] };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', async ({}, testInfo) => {
+        await new Promise(f => setTimeout(f, 1500));
+      });
+    `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('Timeout of 500ms exceeded.');
+});
+
+test('should prioritize command line timeout over project timeout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [{ timeout: 10000}] };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', async ({}, testInfo) => {
+        await new Promise(f => setTimeout(f, 1500));
+      });
+    `
+  }, { timeout: '500' });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('Timeout of 500ms exceeded.');
+});
+
+test('should read config from --config, resolve relative testDir', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'my.config.ts': `
+      import * as path from 'path';
+      module.exports = {
+        testDir: 'dir',
+      };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('ignored', async ({}) => {
+      });
+    `,
+    'dir/b.test.ts': `
+      const { test } = folio;
+      test('run', async ({}) => {
+      });
+    `,
+  }, { config: 'my.config.ts' });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.report.suites.length).toBe(1);
+  expect(result.report.suites[0].file).toBe('b.test.ts');
+});
+
+test('should default testDir to the config file', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'dir/my.config.ts': `
+      module.exports = {};
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('ignored', async ({}) => {
+      });
+    `,
+    'dir/b.test.ts': `
+      const { test } = folio;
+      test('run', async ({}) => {
+      });
+    `,
+  }, { config: path.join('dir', 'my.config.ts') });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.report.suites.length).toBe(1);
+  expect(result.report.suites[0].file).toBe('b.test.ts');
+});
+
+test('should be able to set reporters', async ({ runInlineTest }, testInfo) => {
+  const reportFile = testInfo.outputPath('my-report.json');
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: [
+          ['json', { outputFile: ${JSON.stringify(reportFile)} }],
+          ['list'],
+        ]
+      };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', async () => {
+      });
+    `
+  }, { reporter: '' });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  const report = JSON.parse(fs.readFileSync(reportFile).toString());
+  expect(report.suites[0].file).toBe('a.test.ts');
+});
+
+test('should support different testDirs', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = { projects: [
+        { testDir: __dirname },
+        { testDir: 'dir' },
+      ] };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('runs once', async ({}) => {
+      });
+    `,
+    'dir/b.test.ts': `
+      const { test } = folio;
+      test('runs twice', async ({}) => {
+      });
+    `,
+  });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+
+  expect(result.report.suites[0].specs[0].tests.length).toBe(1);
+  expect(result.report.suites[0].specs[0].title).toBe('runs once');
+
+  expect(result.report.suites[1].specs[0].tests.length).toBe(2);
+  expect(result.report.suites[1].specs[0].title).toBe('runs twice');
+});
+
+test('should allow export default form the config file', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      export default { timeout: 1000 };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('fails', async ({}, testInfo) => {
+        await new Promise(f => setTimeout(f, 2000));
+      });
+    `
+  });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('Timeout of 1000ms exceeded.');
+});
+
+test('should allow root testDir and use it for relative paths', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'config/config.ts': `
+      import * as path from 'path';
+      module.exports = {
+        testDir: path.join(__dirname, '..'),
+        projects: [{ testDir: path.join(__dirname, '..', 'dir') }]
+      };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('fails', async ({}, testInfo) => {
+        expect(1 + 1).toBe(3);
+      });
+    `,
+    'dir/a.test.ts': `
+      const { test } = folio;
+      test('fails', async ({}, testInfo) => {
+        expect(1 + 1).toBe(3);
+      });
+    `,
+  }, { config: path.join('config', 'config.ts') });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.skipped).toBe(0);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain(`1) ${path.join('dir', 'a.test.ts')}:6:7 › fails`);
+});
+
+test('should throw when test() is called in config file', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      folio.test('hey', () => {});
+      module.exports = {};
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('test', async ({}) => {
+      });
+    `,
+  });
+  expect(result.output).toContain('test() can only be called in a test file');
+});
+
+test('should filter by project, case-insensitive', async ({ runInlineTest }) => {
+  const { passed, failed, output, skipped } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'suite1' },
+        { name: 'suite2' },
+      ] };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', async ({}, testInfo) => {
+        console.log(testInfo.project.name);
+      });
+    `
+  }, { project: 'SUite2' });
+  expect(passed).toBe(1);
+  expect(failed).toBe(0);
+  expect(skipped).toBe(0);
+  expect(output).toContain('suite2');
+  expect(output).not.toContain('suite1');
+});
+
+test('should print nice error when project is unknown', async ({ runInlineTest }) => {
+  const { output, exitCode } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'suite1' },
+        { name: 'suite2' },
+      ] };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', async ({}, testInfo) => {
+        console.log(testInfo.project.name);
+      });
+    `
+  }, { project: 'suite3' });
+  expect(exitCode).toBe(1);
+  expect(output).toContain('Project "suite3" not found. Available named projects: "suite1", "suite2"');
+});
+
+test('should work without config file', async ({ runInlineTest }) => {
+  const { exitCode, passed, failed, skipped } = await runInlineTest({
+    'playwright.config.ts': `
+      throw new Error('This file should not be required');
+    `,
+    'dir/a.test.ts': `
+      const { test } = folio;
+      test('pass', async ({}) => {
+        test.expect(1 + 1).toBe(2);
+      });
+    `
+  }, { config: 'dir' });
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(1);
+  expect(failed).toBe(0);
+  expect(skipped).toBe(0);
+});
+
+test('should inerhit use options in projects', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        use: { foo: 'config' },
+        projects: [{
+          use: { bar: 'project' },
+        }]
+      };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', async ({ foo, bar  }, testInfo) => {
+        test.expect(foo).toBe('config');
+        test.expect(bar).toBe('project');
+      });
+    `
+  });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should work with undefined values and base', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        updateSnapshots: undefined,
+      };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', async ({}, testInfo) => {
+        expect(testInfo.config.updateSnapshots).toBe('none');
+      });
+    `
+  }, {}, { CI: '1' });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should work with custom reporter', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'reporter.ts': `
+      class Reporter {
+        constructor(options) {
+          this.options = options;
+        }
+        onBegin() {
+          console.log('\\n%%reporter-begin%%' + this.options.begin);
+        }
+        onTestBegin() {
+          console.log('\\n%%reporter-testbegin%%');
+        }
+        onStdOut() {
+          console.log('\\n%%reporter-stdout%%');
+        }
+        onStdErr() {
+          console.log('\\n%%reporter-stderr%%');
+        }
+        onTestEnd() {
+          console.log('\\n%%reporter-testend%%');
+        }
+        onTimeout() {
+          console.log('\\n%%reporter-timeout%%');
+        }
+        onError() {
+          console.log('\\n%%reporter-error%%');
+        }
+        onEnd() {
+          console.log('\\n%%reporter-end%%' + this.options.end);
+        }
+      }
+      export default Reporter;
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: [
+          [ './reporter.ts', { begin: 'begin', end: 'end' } ]
+        ]
+      };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', async ({}) => {
+        console.log('log');
+        console.error('error');
+      });
+    `
+  }, { reporter: '' });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%reporter-begin%%begin',
+    '%%reporter-testbegin%%',
+    '%%reporter-stdout%%',
+    '%%reporter-stderr%%',
+    '%%reporter-testend%%',
+    '%%reporter-end%%end',
+  ]);
+});

tests/playwright-test/dot-reporter.spec.ts

@@ -0,0 +1,92 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import colors from 'colors/safe';
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+
+test('render expected', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}) => {
+        expect(1).toBe(1);
+      });
+    `,
+  });
+  expect(result.output).toContain(colors.green('·'));
+  expect(result.exitCode).toBe(0);
+});
+
+test('render unexpected', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}) => {
+        expect(1).toBe(0);
+      });
+    `,
+  });
+  expect(result.output).toContain(colors.red('F'));
+  expect(result.exitCode).toBe(1);
+});
+
+test('render unexpected after retry', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}) => {
+        expect(1).toBe(0);
+      });
+    `,
+  }, { retries: 3 });
+  const text = stripAscii(result.output);
+  expect(text).toContain('×××F');
+  expect(result.output).toContain(colors.red('F'));
+  expect(result.exitCode).toBe(1);
+});
+
+test('render flaky', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(3);
+      });
+    `,
+  }, { retries: 3 });
+  const text = stripAscii(result.output);
+  expect(text).toContain('×××±');
+  expect(result.output).toContain(colors.yellow('±'));
+  expect(text).toContain('1 flaky');
+  expect(text).not.toContain('Retry #1');
+  expect(result.exitCode).toBe(0);
+});
+
+test('should work from config', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { reporter: 'dot' };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}) => {
+        expect(1).toBe(1);
+      });
+    `,
+  });
+  expect(result.output).toContain(colors.green('·'));
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/exit-code.spec.ts

@@ -0,0 +1,163 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+
+function monotonicTime(): number {
+  const [seconds, nanoseconds] = process.hrtime();
+  return seconds * 1000 + (nanoseconds / 1000000 | 0);
+}
+
+test('should collect stdio', async ({ runInlineTest }) => {
+  const { exitCode, report } = await runInlineTest({
+    'stdio.spec.js': `
+      const { test } = folio;
+      test('stdio', () => {
+        process.stdout.write('stdout text');
+        process.stdout.write(Buffer.from('stdout buffer'));
+        process.stderr.write('stderr text');
+        process.stderr.write(Buffer.from('stderr buffer'));
+      });
+    `
+  });
+  expect(exitCode).toBe(0);
+  const testResult = report.suites[0].specs[0].tests[0].results[0];
+  const { stdout, stderr } = testResult;
+  expect(stdout).toEqual([{ text: 'stdout text' }, { buffer: Buffer.from('stdout buffer').toString('base64') }]);
+  expect(stderr).toEqual([{ text: 'stderr text' }, { buffer: Buffer.from('stderr buffer').toString('base64') }]);
+});
+
+test('should work with not defined errors', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'is-not-defined-error.spec.ts': `
+      foo();
+    `
+  });
+  expect(stripAscii(result.output)).toContain('foo is not defined');
+  expect(result.exitCode).toBe(1);
+});
+
+test('should work with typescript', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'global-foo.js': `
+      global.foo = true;
+      module.exports = {
+        abc: 123
+      };
+    `,
+    'typescript.spec.ts': `
+      import './global-foo';
+
+      const { test } = folio;
+      test('should find global foo', () => {
+        expect(global['foo']).toBe(true);
+      });
+
+      test('should work with type annotations', () => {
+        const x: number = 5;
+        expect(x).toBe(5);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should repeat each', async ({ runInlineTest }) => {
+  const { exitCode, report, passed } = await runInlineTest({
+    'one-success.spec.js': `
+      const { test } = folio;
+      test('succeeds', () => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { 'repeat-each': 3 });
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(3);
+  expect(report.suites.length).toBe(1);
+  expect(report.suites[0].specs.length).toBe(1);
+  expect(report.suites[0].specs[0].tests.length).toBe(3);
+});
+
+test('should allow flaky', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('flake', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `,
+  }, { retries: 1 });
+  expect(result.exitCode).toBe(0);
+  expect(result.flaky).toBe(1);
+});
+
+test('should fail on unexpected pass', async ({ runInlineTest }) => {
+  const { exitCode, failed, output } = await runInlineTest({
+    'unexpected-pass.spec.js': `
+      const { test } = folio;
+      test('succeeds', () => {
+        test.fail();
+        expect(1 + 1).toBe(2);
+      });
+    `
+  });
+  expect(exitCode).toBe(1);
+  expect(failed).toBe(1);
+  expect(output).toContain('passed unexpectedly');
+});
+
+test('should respect global timeout', async ({ runInlineTest }) => {
+  const now = monotonicTime();
+  const { exitCode, output } = await runInlineTest({
+    'one-timeout.spec.js': `
+      const { test } = folio;
+      test('timeout', async () => {
+        await new Promise(f => setTimeout(f, 10000));
+      });
+    `
+  }, { 'timeout': 100000, 'global-timeout': 3000 });
+  expect(exitCode).toBe(1);
+  expect(output).toContain('Timed out waiting 3s for the entire test run');
+  expect(monotonicTime() - now).toBeGreaterThan(2900);
+});
+
+test('should exit with code 1 if the specified folder does not exist', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { testDir: '111111111111.js' };
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`111111111111.js does not exist`);
+});
+
+test('should exit with code 1 if passed a file name', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { testDir: 'test.spec.js' };
+    `,
+    'test.spec.js': `
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`test.spec.js is not a directory`);
+});
+
+test('should exit with code 1 when config is not found', async ({runInlineTest}) => {
+  const result = await runInlineTest({'my.config.js': ''}, { 'config': 'foo.config.js' });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`foo.config.js does not exist`);
+});

tests/playwright-test/expect.spec.ts

@@ -0,0 +1,126 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should be able to extend the expect matchers with test.extend in the folio config', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      folio.expect.extend({
+        toBeWithinRange(received, floor, ceiling) {
+          const pass = received >= floor && received <= ceiling;
+          if (pass) {
+            return {
+              message: () =>
+                'passed',
+              pass: true,
+            };
+          } else {
+            return {
+              message: () => 'failed',
+              pass: false,
+            };
+          }
+        },
+      });
+      export const test = folio.test;
+    `,
+    'expect-test.spec.ts': `
+      import { test } from './helper';
+      test('numeric ranges', () => {
+        test.expect(100).toBeWithinRange(90, 110);
+        test.expect(101).not.toBeWithinRange(0, 100);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should work with default expect prototype functions', async ({runTSC}) => {
+  const result = await runTSC({
+    'a.spec.ts': `
+      const { test } = folio;
+      const expected = [1, 2, 3, 4, 5, 6];
+      test.expect([4, 1, 6, 7, 3, 5, 2, 5, 4, 6]).toEqual(
+        expect.arrayContaining(expected),
+      );
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should work with default expect matchers', async ({runTSC}) => {
+  const result = await runTSC({
+    'a.spec.ts': `
+      const { test } = folio;
+      test.expect(42).toBe(42);
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should work with jest-community/jest-extended', async ({runTSC}) => {
+  const result = await runTSC({
+    'global.d.ts': `
+      // Extracted example from their typings.
+      // Reference: https://github.com/jest-community/jest-extended/blob/master/types/index.d.ts
+      declare namespace jest {
+        interface Matchers<R> {
+          toBeEmpty(): R;
+        }
+      }
+    `,
+    'a.spec.ts': `
+      const { test } = folio;
+      test.expect('').toBeEmpty();
+      test.expect('hello').not.toBeEmpty();
+      test.expect([]).toBeEmpty();
+      test.expect(['hello']).not.toBeEmpty();
+      test.expect({}).toBeEmpty();
+      test.expect({ hello: 'world' }).not.toBeEmpty();
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should work with custom folio namespace', async ({runTSC}) => {
+  const result = await runTSC({
+    'global.d.ts': `
+      // Extracted example from their typings.
+      // Reference: https://github.com/jest-community/jest-extended/blob/master/types/index.d.ts
+      declare namespace folio {
+        interface Matchers<R> {
+          toBeEmpty(): R;
+        }
+      }
+    `,
+    'a.spec.ts': `
+      const { test } = folio;
+      test.expect.extend({
+        toBeWithinRange() { },
+      });
+
+      test.expect('').toBeEmpty();
+      test.expect('hello').not.toBeEmpty();
+      test.expect([]).toBeEmpty();
+      test.expect(['hello']).not.toBeEmpty();
+      test.expect({}).toBeEmpty();
+      test.expect({ hello: 'world' }).not.toBeEmpty();
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/fixture-errors.spec.ts

@@ -0,0 +1,392 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+
+test('should handle fixture timeout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const test = folio.test.extend({
+        timeout: async ({}, runTest) => {
+          await runTest();
+          await new Promise(f => setTimeout(f, 100000));
+        }
+      });
+
+      test('fixture timeout', async ({timeout}) => {
+        expect(1).toBe(1);
+      });
+
+      test('failing fixture timeout', async ({timeout}) => {
+        expect(1).toBe(2);
+      });
+    `
+  }, { timeout: 500 });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Timeout of 500ms');
+  expect(result.failed).toBe(2);
+});
+
+test('should handle worker fixture timeout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const test = folio.test.extend({
+        timeout: [async ({}, runTest) => {
+          await runTest();
+          await new Promise(f => setTimeout(f, 100000));
+        }, { scope: 'worker' }]
+      });
+
+      test('fails', async ({timeout}) => {
+      });
+    `
+  }, { timeout: 500 });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Timeout of 500ms');
+});
+
+test('should handle worker fixture error', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const test = folio.test.extend({
+        failure: [async ({}, runTest) => {
+          throw new Error('Worker failed');
+        }, { scope: 'worker' }]
+      });
+
+      test('fails', async ({failure}) => {
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('Worker failed');
+});
+
+test('should handle worker tear down fixture error', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const test = folio.test.extend({
+        failure: [async ({}, runTest) => {
+          await runTest();
+          throw new Error('Worker failed');
+        }, { scope: 'worker' }]
+      });
+
+      test('pass', async ({failure}) => {
+        expect(true).toBe(true);
+      });
+    `
+  });
+  expect(result.report.errors[0].message).toContain('Worker failed');
+  expect(result.exitCode).toBe(1);
+});
+
+test('should throw when using non-defined super worker fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const test = folio.test.extend({
+        foo: [async ({ foo }, runTest) => {
+          await runTest();
+        }, { scope: 'worker' }]
+      });
+
+      test('works', async ({foo}) => {});
+    `
+  });
+  expect(result.output).toContain(`Fixture "foo" references itself, but does not have a base implementation.`);
+  expect(result.output).toContain('a.spec.ts:5:31');
+  expect(result.exitCode).toBe(1);
+});
+
+test('should throw when defining test fixture with the same name as a worker fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'e.spec.ts': `
+      const test1 = folio.test.extend({
+        foo: [async ({}, runTest) => {
+          await runTest();
+        }, { scope: 'worker' }]
+      });
+      const test2 = test1.extend({
+        foo: [async ({}, runTest) => {
+          await runTest();
+        }, { scope: 'test' }]
+      });
+
+      test2('works', async ({foo}) => {});
+    `,
+  });
+  expect(result.output).toContain(`Fixture "foo" has already been registered as a { scope: 'worker' } fixture.`);
+  expect(result.output).toContain(`e.spec.ts:10`);
+  expect(result.output).toContain(`e.spec.ts:5`);
+  expect(result.exitCode).toBe(1);
+});
+
+test('should throw when defining worker fixture with the same name as a test fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'e.spec.ts': `
+      const test1 = folio.test.extend({
+        foo: [async ({}, runTest) => {
+          await runTest();
+        }, { scope: 'test' }]
+      });
+      const test2 = test1.extend({
+        foo: [async ({}, runTest) => {
+          await runTest();
+        }, { scope: 'worker' }]
+      });
+
+      test2('works', async ({foo}) => {});
+    `,
+  });
+  expect(result.output).toContain(`Fixture "foo" has already been registered as a { scope: 'test' } fixture.`);
+  expect(result.output).toContain(`e.spec.ts:10`);
+  expect(result.output).toContain(`e.spec.ts:5`);
+  expect(result.exitCode).toBe(1);
+});
+
+test('should throw when worker fixture depends on a test fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'f.spec.ts': `
+      const test = folio.test.extend({
+        foo: [async ({}, runTest) => {
+          await runTest();
+        }, { scope: 'test' }],
+
+        bar: [async ({ foo }, runTest) => {
+          await runTest();
+        }, { scope: 'worker' }],
+      });
+
+      test('works', async ({bar}) => {});
+    `,
+  });
+  expect(result.output).toContain('Worker fixture "bar" cannot depend on a test fixture "foo".');
+  expect(result.output).toContain(`f.spec.ts:5`);
+  expect(result.exitCode).toBe(1);
+});
+
+test('should throw when beforeAll hook depends on a test fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'f.spec.ts': `
+      const test = folio.test.extend({
+        foo: [async ({}, runTest) => {
+          await runTest();
+        }, { scope: 'test' }],
+      });
+
+      test.beforeAll(async ({ foo }) => {});
+      test('works', async ({ foo }) => {});
+    `,
+  });
+  expect(result.output).toContain('beforeAll hook cannot depend on a test fixture "foo".');
+  expect(result.output).toContain(`f.spec.ts:11:12`);
+  expect(result.output).toContain(`f.spec.ts:5:31`);
+  expect(result.exitCode).toBe(1);
+});
+
+test('should throw when afterAll hook depends on a test fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'f.spec.ts': `
+      const test = folio.test.extend({
+        foo: [async ({}, runTest) => {
+          await runTest();
+        }, { scope: 'test' }],
+      });
+
+      test.afterAll(async ({ foo }) => {});
+      test('works', async ({ foo }) => {});
+    `,
+  });
+  expect(result.output).toContain('afterAll hook cannot depend on a test fixture "foo".');
+  expect(result.output).toContain(`f.spec.ts:11:12`);
+  expect(result.output).toContain(`f.spec.ts:5:31`);
+  expect(result.exitCode).toBe(1);
+});
+
+test('should define the same fixture in two files', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const test1 = folio.test.extend({
+        foo: [async ({}, runTest) => {
+          await runTest();
+        }, { scope: 'worker' }]
+      });
+
+      test1('works', async ({foo}) => {});
+    `,
+    'b.spec.ts': `
+      const test2 = folio.test.extend({
+        foo: [async ({}, runTest) => {
+          await runTest();
+        }, { scope: 'worker' }]
+      });
+
+      test2('works', async ({foo}) => {});
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+});
+
+test('should detect fixture dependency cycle', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'x.spec.ts': `
+      const test = folio.test.extend({
+        good1: async ({}, run) => run(),
+        foo: async ({bar}, run) => run(),
+        bar: async ({baz}, run) => run(),
+        good2: async ({good1}, run) => run(),
+        baz: async ({qux}, run) => run(),
+        qux: async ({foo}, run) => run(),
+      });
+
+      test('works', async ({foo}) => {});
+    `,
+  });
+  expect(result.output).toContain('Fixtures "bar" -> "baz" -> "qux" -> "foo" -> "bar" form a dependency cycle.');
+  expect(result.output).toContain('"foo" defined at');
+  expect(result.output).toContain('"bar" defined at');
+  expect(result.output).toContain('"baz" defined at');
+  expect(result.output).toContain('"qux" defined at');
+  expect(result.output).toContain('x.spec.ts:5:31');
+  expect(result.exitCode).toBe(1);
+});
+
+test('should not reuse fixtures from one file in another one', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const test = folio.test.extend({ foo: ({}, run) => run() });
+      test('test1', async ({}) => {});
+    `,
+    'b.spec.ts': `
+      const test = folio.test;
+      test('test1', async ({}) => {});
+      test('test2', async ({foo}) => {});
+    `,
+  });
+  expect(result.output).toContain('Test has unknown parameter "foo".');
+  expect(result.output).toContain('b.spec.ts:7:7');
+});
+
+test('should throw for cycle in two overrides', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const test1 = folio.test.extend({
+        foo: async ({}, run) => await run('foo'),
+        bar: async ({}, run) => await run('bar'),
+      });
+      const test2 = test1.extend({
+        foo: async ({ foo, bar }, run) => await run(foo + '-' + bar),
+      });
+      const test3 = test2.extend({
+        bar: async ({ bar, foo }, run) => await run(bar + '-' + foo),
+      });
+
+      test3('test', async ({foo, bar}) => {
+        expect(1).toBe(1);
+      });
+    `,
+  });
+  expect(result.output).toContain('Fixtures "bar" -> "foo" -> "bar" form a dependency cycle.');
+  expect(result.output).toContain('a.test.js:9');
+  expect(result.output).toContain('a.test.js:12');
+});
+
+test('should throw when overridden worker fixture depends on a test fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'f.spec.ts': `
+      const test1 = folio.test.extend({
+        foo: async ({}, run) => await run('foo'),
+        bar: [ async ({}, run) => await run('bar'), { scope: 'worker' } ],
+      });
+      const test2 = test1.extend({
+        bar: async ({ foo }, run) => await run(),
+      });
+
+      test2('works', async ({bar}) => {});
+    `,
+  });
+  expect(result.output).toContain('Worker fixture "bar" cannot depend on a test fixture "foo".');
+  expect(result.exitCode).toBe(1);
+});
+
+test('should throw for unknown fixture parameter', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'f.spec.ts': `
+      const test = folio.test.extend({
+        foo: async ({ bar }, run) => await run('foo'),
+      });
+
+      test('works', async ({ foo }) => {});
+    `,
+  });
+  expect(result.output).toContain('Fixture "foo" has unknown parameter "bar".');
+  expect(result.output).toContain('f.spec.ts:5:31');
+  expect(result.exitCode).toBe(1);
+});
+
+test('should throw when calling runTest twice', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'f.spec.ts': `
+      const test = folio.test.extend({
+        foo: async ({}, run) => {
+          await run();
+          await run();
+        }
+      });
+
+      test('works', async ({foo}) => {});
+    `,
+  });
+  expect(result.results[0].error.message).toBe('Cannot provide fixture value for the second time');
+  expect(result.exitCode).toBe(1);
+});
+
+test('should print nice error message for problematic fixtures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'x.spec.ts': `
+      const test = folio.test.extend({
+        bad: [ undefined, { get scope() { throw new Error('oh my!') } } ],
+      });
+      test('works', async ({foo}) => {});
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('oh my!');
+  expect(result.output).toContain('x.spec.ts:5:31');
+});
+
+test('should exit with timeout when fixture causes an exception in the test', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const test = folio.test.extend({
+        throwAfterTimeout: async ({}, use) => {
+          let callback;
+          const promise = new Promise((f, r) => callback = r);
+          await use(promise);
+          callback(new Error('BAD'));
+        },
+      });
+      test('times out and throws', async ({ throwAfterTimeout }) => {
+        await throwAfterTimeout;
+      });
+    `,
+  }, { timeout: 500 });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('Timeout of 500ms exceeded');
+});

tests/playwright-test/fixtures.spec.ts

@@ -0,0 +1,598 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should work', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        asdf: async ({}, test) => await test(123),
+      });
+
+      test('should use asdf', async ({asdf}) => {
+        expect(asdf).toBe(123);
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+});
+
+test('should work with a sync test function', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        asdf: async ({}, test) => await test(123),
+      });
+
+      test('should use asdf', ({asdf}) => {
+        expect(asdf).toBe(123);
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+});
+
+test('should work with a sync fixture function', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        asdf: ({}, use) => {
+          use(123);
+        },
+      });
+
+      test('should use asdf', ({asdf}) => {
+        expect(asdf).toBe(123);
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+});
+
+test('should work with a non-arrow function', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        asdf: async ({}, test) => await test(123),
+      });
+
+      test('should use asdf', function ({asdf}) {
+        expect(asdf).toBe(123);
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+});
+
+test('should work with a named function', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        asdf: async ({}, test) => await test(123),
+      });
+
+      test('should use asdf', async function hello({asdf}) {
+        expect(asdf).toBe(123);
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+});
+
+test('should work with renamed parameters', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        asdf: async ({}, test) => await test(123),
+      });
+
+      test('should use asdf', function ({asdf: renamed}) {
+        expect(renamed).toBe(123);
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+});
+
+test('should work with destructured object', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        asdf: async ({}, test) => await test({ foo: 'foo', bar: { x: 'x', y: 'y' }, baz: 'baz' }),
+      });
+
+      test('should use asdf', async ({ asdf: { foo,
+          bar: { x, y }, baz } }) => {
+        expect(foo).toBe('foo');
+        expect(x).toBe('x');
+        expect(y).toBe('y');
+        expect(baz).toBe('baz');
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+});
+
+test('should work with destructured array', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        asdf: async ({}, test) => await test(['foo', 'bar', { baz: 'baz' }]),
+        more: async ({}, test) => await test(55),
+      });
+
+      test('should use asdf', async (
+
+        {
+          asdf: [foo, bar,        { baz}]
+
+
+          ,more}) => {
+        expect(foo).toBe('foo');
+        expect(bar).toBe('bar');
+        expect(baz).toBe('baz');
+        expect(more).toBe(55);
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+});
+
+test('should fail if parameters are not destructured', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        asdf: async ({}, test) => await test(123),
+      });
+      test('should pass', function () {
+        expect(1).toBe(1);
+      });
+      test('should use asdf', function (abc) {
+        expect(abc.asdf).toBe(123);
+      });
+    `,
+  });
+  expect(result.output).toContain('First argument must use the object destructuring pattern: abc');
+  expect(result.output).toContain('a.test.js:11:7');
+  expect(result.results.length).toBe(0);
+});
+
+test('should fail with an unknown fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      folio.test('should use asdf', async ({asdf}) => {
+        expect(asdf).toBe(123);
+      });
+    `,
+  });
+  expect(result.output).toContain('Test has unknown parameter "asdf".');
+  expect(result.output).toContain('a.test.js:5:13');
+  expect(result.results.length).toBe(0);
+});
+
+test('should run the fixture every time', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      let counter = 0;
+      const test = folio.test.extend({
+        asdf: async ({}, test) => await test(counter++),
+      });
+      test('should use asdf', async ({asdf}) => {
+        expect(asdf).toBe(0);
+      });
+      test('should use asdf', async ({asdf}) => {
+        expect(asdf).toBe(1);
+      });
+      test('should use asdf', async ({asdf}) => {
+        expect(asdf).toBe(2);
+      });
+    `,
+  });
+  expect(results.map(r => r.status)).toEqual(['passed', 'passed', 'passed']);
+});
+
+test('should only run worker fixtures once', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      let counter = 0;
+      const test = folio.test.extend({
+        asdf: [ async ({}, test) => await test(counter++), { scope: 'worker' } ],
+      });
+      test('should use asdf', async ({asdf}) => {
+        expect(asdf).toBe(0);
+      });
+      test('should use asdf', async ({asdf}) => {
+        expect(asdf).toBe(0);
+      });
+      test('should use asdf', async ({asdf}) => {
+        expect(asdf).toBe(0);
+      });
+    `,
+  });
+  expect(results.map(r => r.status)).toEqual(['passed', 'passed', 'passed']);
+});
+
+test('each file should get their own fixtures', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'a.test.js': `
+      const test = folio.test.extend({
+        worker: [ async ({}, test) => await test('worker-a'), { scope: 'worker' } ],
+        test: async ({}, test) => await test('test-a'),
+      });
+      test('should use worker', async ({worker, test}) => {
+        expect(worker).toBe('worker-a');
+        expect(test).toBe('test-a');
+      });
+    `,
+    'b.test.js': `
+      const test = folio.test.extend({
+        worker: [ async ({}, test) => await test('worker-b'), { scope: 'worker' } ],
+        test: async ({}, test) => await test('test-b'),
+      });
+      test('should use worker', async ({worker, test}) => {
+        expect(worker).toBe('worker-b');
+        expect(test).toBe('test-b');
+      });
+    `,
+    'c.test.js': `
+      const test = folio.test.extend({
+        worker: [ async ({}, test) => await test('worker-c'), { scope: 'worker' } ],
+        test: async ({}, test) => await test('test-c'),
+      });
+      test('should use worker', async ({worker, test}) => {
+        expect(worker).toBe('worker-c');
+        expect(test).toBe('test-c');
+      });
+    `,
+  });
+  expect(results.map(r => r.status)).toEqual(['passed', 'passed', 'passed']);
+});
+
+test('tests should be able to share worker fixtures', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'worker.js': `
+      global.counter = 0;
+      const test = folio.test.extend({
+        worker: [ async ({}, test) => await test(global.counter++), { scope: 'worker' } ],
+      });
+      module.exports = test;
+    `,
+    'a.test.js': `
+      const test = require('./worker.js');
+      test('should use worker', async ({worker}) => {
+        expect(worker).toBe(0);
+      });
+    `,
+    'b.test.js': `
+      const test = require('./worker.js');
+      test('should use worker', async ({worker}) => {
+        expect(worker).toBe(0);
+      });
+    `,
+    'c.test.js': `
+      const test = require('./worker.js');
+      test('should use worker', async ({worker}) => {
+        expect(worker).toBe(0);
+      });
+    `,
+  });
+  expect(results.map(r => r.status)).toEqual(['passed', 'passed', 'passed']);
+});
+
+test('automatic fixtures should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      let counterTest = 0;
+      let counterWorker = 0;
+      const test = folio.test;
+      test.use({
+        automaticTestFixture: [ async ({}, runTest) => {
+          ++counterTest;
+          await runTest();
+        }, { auto: true } ],
+
+        automaticWorkerFixture: [ async ({}, runTest) => {
+          ++counterWorker;
+          await runTest();
+        }, { scope: 'worker', auto: true } ],
+      });
+      test.beforeAll(async ({}) => {
+        expect(counterWorker).toBe(1);
+        expect(counterTest).toBe(0);
+      });
+      test.beforeEach(async ({}) => {
+        expect(counterWorker).toBe(1);
+        expect(counterTest === 1 || counterTest === 2).toBe(true);
+      });
+      test('test 1', async ({}) => {
+        expect(counterWorker).toBe(1);
+        expect(counterTest).toBe(1);
+      });
+      test('test 2', async ({}) => {
+        expect(counterWorker).toBe(1);
+        expect(counterTest).toBe(2);
+      });
+      test.afterEach(async ({}) => {
+        expect(counterWorker).toBe(1);
+        expect(counterTest === 1 || counterTest === 2).toBe(true);
+      });
+      test.afterAll(async ({}) => {
+        expect(counterWorker).toBe(1);
+        expect(counterTest).toBe(2);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.results.map(r => r.status)).toEqual(['passed', 'passed']);
+});
+
+test('tests does not run non-automatic worker fixtures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      let counter = 0;
+      const test = folio.test.extend({
+        nonAutomaticWorkerFixture: [ async ({}, runTest) => {
+          ++counter;
+          await runTest();
+        }, { scope: 'worker' }],
+      });
+      test('test 1', async ({}) => {
+        expect(counter).toBe(0);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.results.map(r => r.status)).toEqual(['passed']);
+});
+
+test('should teardown fixtures after timeout', async ({ runInlineTest }, testInfo) => {
+  const file = testInfo.outputPath('log.txt');
+  require('fs').writeFileSync(file, '', 'utf8');
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const test = folio.test.extend({
+        file: [ ${JSON.stringify(file)}, { scope: 'worker' } ],
+        w: [ async ({ file }, runTest) => {
+          await runTest('w');
+          require('fs').appendFileSync(file, 'worker fixture teardown\\n', 'utf8');
+        }, { scope: 'worker' } ],
+        t: async ({ file }, runTest) => {
+          await runTest('t');
+          require('fs').appendFileSync(file, 'test fixture teardown\\n', 'utf8');
+        },
+      });
+      test('test', async ({t, w}) => {
+        expect(t).toBe('t');
+        expect(w).toBe('w');
+        await new Promise(() => {});
+      });
+    `,
+  }, { timeout: 1000 });
+  expect(result.results[0].status).toBe('timedOut');
+  const content = require('fs').readFileSync(file, 'utf8');
+  expect(content).toContain('worker fixture teardown');
+  expect(content).toContain('test fixture teardown');
+});
+
+test('should work with two different test objects', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const test1 = folio.test.extend({
+        foo: async ({}, test) => await test(123),
+      });
+      const test2 = folio.test.extend({
+        bar: async ({}, test) => await test(456),
+      });
+      test1('test 1', async ({foo}) => {
+        expect(foo).toBe(123);
+      });
+      test2('test 2', async ({bar}) => {
+        expect(bar).toBe(456);
+      });
+    `,
+  });
+  expect(result.results.map(r => r.workerIndex).sort()).toEqual([0, 0]);
+  expect(result.results.map(r => r.status).sort()).toEqual(['passed', 'passed']);
+});
+
+test('should work with overrides calling base', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const test1 = folio.test.extend({
+        dep: async ({}, test) => await test('override'),
+        foo: async ({}, test) => await test('base'),
+        bar: async ({foo}, test) => await test(foo + '-bar'),
+      });
+      const test2 = test1.extend({
+        foo: async ({ foo, dep }, test) => await test(foo + '-' + dep + '1'),
+      });
+      const test3 = test2.extend({
+        foo: async ({ foo, dep }, test) => await test(foo + '-' + dep + '2'),
+      });
+      test3('test', async ({bar}) => {
+        expect(bar).toBe('base-override1-override2-bar');
+      });
+    `,
+  });
+  expect(result.results[0].status).toBe('passed');
+});
+
+test('should understand worker fixture params in overrides calling base', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const test1 = folio.test.extend({
+        param: [ 'param', { scope: 'worker' }],
+        foo: async ({}, test) => await test('foo'),
+        bar: async ({foo}, test) => await test(foo + '-bar'),
+      });
+      const test2 = test1.extend({
+        foo: async ({ foo, param }, test) => await test(foo + '-' + param),
+      });
+      const test3 = test2.extend({
+        foo: async ({ foo }, test) => await test(foo + '-override'),
+      });
+      test3('test', async ({ bar }) => {
+        console.log(bar);
+      });
+    `,
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { use: { param: 'p1' } },
+        { use: { param: 'p2' } },
+        { use: { param: 'p3' } },
+      ]};
+    `,
+  });
+  const outputs = result.results.map(r => r.stdout[0].text.replace(/\s/g, ''));
+  expect(outputs.sort()).toEqual(['foo-p1-override-bar', 'foo-p2-override-bar', 'foo-p3-override-bar']);
+});
+
+test('should work with two overrides calling base', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const test1 = folio.test.extend({
+        foo: async ({}, test) => await test('foo'),
+        bar: async ({}, test) => await test('bar'),
+        baz: async ({foo, bar}, test) => await test(foo + '-baz-' + bar),
+      });
+      const test2 = test1.extend({
+        foo: async ({ foo, bar }, test) => await test(foo + '-' + bar),
+        bar: async ({ bar }, test) => await test(bar + '-override'),
+      });
+      test2('test', async ({baz}) => {
+        expect(baz).toBe('foo-bar-override-baz-bar-override');
+      });
+    `,
+  });
+  expect(result.results[0].status).toBe('passed');
+});
+
+test('should not create a new worker for test fixtures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = folio;
+      test('base test', async ({}, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+      });
+
+      const test2 = test.extend({
+        foo: async ({}, run) => {
+          console.log('foo-a');
+          await run();
+        }
+      });
+      test2('a test', async ({ foo }, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+      });
+    `,
+    'b.test.ts': `
+      const { test } = folio;
+      const test2 = test.extend({
+        foo: async ({}, run) => {
+          console.log('foo-b');
+          await run();
+        }
+      });
+      const test3 = test2.extend({
+        foo: async ({ foo }, run) => {
+          console.log('foo-c');
+          await run();
+        }
+      });
+      test3('b test', async ({ foo }, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+      });
+    `,
+  }, { workers: 1 });
+  expect(result.output).toContain('foo-a');
+  expect(result.output).toContain('foo-b');
+  expect(result.output).toContain('foo-c');
+  expect(result.passed).toBe(3);
+});
+
+test('should create a new worker for worker fixtures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = folio;
+      test('base test', async ({}, testInfo) => {
+        expect(testInfo.workerIndex).toBe(1);
+      });
+
+      const test2 = test.extend({
+        foo: [async ({}, run) => {
+          console.log('foo-a');
+          await run();
+        }, { scope: 'worker' }],
+      });
+      test2('a test', async ({ foo }, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+      });
+    `,
+    'b.test.ts': `
+      const { test } = folio;
+      const test2 = test.extend({
+        bar: async ({}, run) => {
+          console.log('bar-b');
+          await run();
+        },
+      });
+      test2('b test', async ({ bar }, testInfo) => {
+        expect(testInfo.workerIndex).toBe(1);
+      });
+    `,
+  }, { workers: 1 });
+  expect(result.output).toContain('foo-a');
+  expect(result.output).toContain('bar-b');
+  expect(result.passed).toBe(3);
+});
+
+test('should run tests in order', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = folio;
+      test('test1', async ({}, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+        console.log('\\n%%test1');
+      });
+
+      const child = test.extend({
+        foo: async ({}, run) => {
+          console.log('\\n%%beforeEach');
+          await run();
+          console.log('\\n%%afterEach');
+        },
+      });
+      child('test2', async ({ foo }, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+        console.log('\\n%%test2');
+      });
+
+      test('test3', async ({}, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+        console.log('\\n%%test3');
+      });
+    `,
+  }, { workers: 1 });
+  expect(result.passed).toBe(3);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%test1',
+    '%%beforeEach',
+    '%%test2',
+    '%%afterEach',
+    '%%test3',
+  ]);
+});

tests/playwright-test/gitignore.spec.ts

@@ -0,0 +1,111 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should respect .gitignore', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    '.gitignore': `a.spec.js`,
+    'a.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'b.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should respect nested .gitignore', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a/.gitignore': `a.spec.js`,
+    'a/a.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'a/b.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should respect enclosing .gitignore', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    '.gitignore': `a/a.spec.js`,
+    'a/a.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'a/b.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should respect negations and comments in .gitignore', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    '.gitignore': `
+      # A comment
+      dir1/
+      /dir2
+      #a.spec.js
+      !dir1/foo/a.spec.js
+    `,
+    'a.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => console.log('\\n%%a.spec.js'));
+    `,
+    'dir1/a.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => console.log('\\n%%dir1/a.spec.js'));
+    `,
+    'dir1/foo/a.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => console.log('\\n%%dir1/foo/a.spec.js'));
+    `,
+    'dir2/a.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => console.log('\\n%%dir2/a.spec.js'));
+    `,
+    'dir3/.gitignore': `
+      b.*.js
+    `,
+    'dir3/a.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => console.log('\\n%%dir3/a.spec.js'));
+    `,
+    'dir3/b.spec.js': `
+      const { test } = folio;
+      test('pass', ({}) => console.log('\\n%%dir3/b.spec.js'));
+    `,
+  }, { workers: 1 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
+    '%%a.spec.js',
+    '%%dir1/foo/a.spec.js',
+    '%%dir3/a.spec.js',
+  ]);
+});

tests/playwright-test/global-setup.spec.ts

@@ -0,0 +1,214 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('globalSetup and globalTeardown should work', async ({ runInlineTest }) => {
+  const { results, output } = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = {
+        globalSetup: path.join(__dirname, 'globalSetup.ts'),
+        globalTeardown: path.join(__dirname, 'globalTeardown.ts'),
+      };
+    `,
+    'globalSetup.ts': `
+      module.exports = async () => {
+        await new Promise(f => setTimeout(f, 100));
+        global.value = 42;
+        process.env.FOO = String(global.value);
+      };
+    `,
+    'globalTeardown.ts': `
+      module.exports = async () => {
+        console.log('teardown=' + global.value);
+      };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('should work', async ({}, testInfo) => {
+        expect(process.env.FOO).toBe('42');
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+  expect(output).toContain('teardown=42');
+});
+
+test('globalTeardown runs after failures', async ({ runInlineTest }) => {
+  const { results, output } = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = {
+        globalSetup: path.join(__dirname, 'globalSetup.ts'),
+        globalTeardown: path.join(__dirname, 'globalTeardown.ts'),
+      };
+    `,
+    'globalSetup.ts': `
+      module.exports = async () => {
+        await new Promise(f => setTimeout(f, 100));
+        global.value = 42;
+        process.env.FOO = String(global.value);
+      };
+    `,
+    'globalTeardown.ts': `
+      module.exports = async () => {
+        console.log('teardown=' + global.value);
+      };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('should work', async ({}, testInfo) => {
+        expect(process.env.FOO).toBe('43');
+      });
+    `,
+  });
+  expect(results[0].status).toBe('failed');
+  expect(output).toContain('teardown=42');
+});
+
+test('globalTeardown does not run when globalSetup times out', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = {
+        globalSetup: path.join(__dirname, 'globalSetup.ts'),
+        globalTeardown: path.join(__dirname, 'globalTeardown.ts'),
+        globalTimeout: 1000,
+      };
+    `,
+    'globalSetup.ts': `
+      module.exports = async () => {
+        await new Promise(f => setTimeout(f, 10000));
+      };
+    `,
+    'globalTeardown.ts': `
+      module.exports = async () => {
+        console.log('teardown=');
+      };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('should not run', async ({}, testInfo) => {
+      });
+    `,
+  });
+  // We did not collect tests, so everything should be zero.
+  expect(result.skipped).toBe(0);
+  expect(result.passed).toBe(0);
+  expect(result.failed).toBe(0);
+  expect(result.exitCode).toBe(1);
+  expect(result.output).not.toContain('teardown=');
+});
+
+test('globalSetup should be run before requiring tests', async ({ runInlineTest }) => {
+  const { passed } = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = {
+        globalSetup: path.join(__dirname, 'globalSetup.ts'),
+      };
+    `,
+    'globalSetup.ts': `
+      module.exports = async () => {
+        process.env.FOO = JSON.stringify({ foo: 'bar' });
+      };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      let value = JSON.parse(process.env.FOO);
+      test('should work', async ({}) => {
+        expect(value).toEqual({ foo: 'bar' });
+      });
+    `,
+  });
+  expect(passed).toBe(1);
+});
+
+test('globalSetup should work with sync function', async ({ runInlineTest }) => {
+  const { passed } = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = {
+        globalSetup: path.join(__dirname, 'globalSetup.ts'),
+      };
+    `,
+    'globalSetup.ts': `
+      module.exports = () => {
+        process.env.FOO = JSON.stringify({ foo: 'bar' });
+      };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      let value = JSON.parse(process.env.FOO);
+      test('should work', async ({}) => {
+        expect(value).toEqual({ foo: 'bar' });
+      });
+    `,
+  });
+  expect(passed).toBe(1);
+});
+
+test('globalSetup should throw when passed non-function', async ({ runInlineTest }) => {
+  const { output } = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = {
+        globalSetup: path.join(__dirname, 'globalSetup.ts'),
+      };
+    `,
+    'globalSetup.ts': `
+      module.exports = 42;
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('should work', async ({}) => {
+      });
+    `,
+  });
+  expect(output).toContain(`globalSetup file must export a single function.`);
+});
+
+test('globalSetup should work with default export and run the returned fn', async ({ runInlineTest }) => {
+  const { output, exitCode, passed } = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = {
+        globalSetup: path.join(__dirname, 'globalSetup.ts'),
+      };
+    `,
+    'globalSetup.ts': `
+      function setup() {
+        let x = 42;
+        console.log('\\n%%setup: ' + x);
+        return async () => {
+          await x;
+          console.log('\\n%%teardown: ' + x);
+        };
+      }
+      export default setup;
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('should work', async ({}) => {
+      });
+    `,
+  });
+  expect(passed).toBe(1);
+  expect(exitCode).toBe(0);
+  expect(output).toContain(`%%setup: 42`);
+  expect(output).toContain(`%%teardown: 42`);
+});

tests/playwright-test/golden.spec.ts

@@ -0,0 +1,256 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import colors from 'colors/safe';
+import * as fs from 'fs';
+import * as path from 'path';
+import { test, expect } from './playwright-test-fixtures';
+
+test('should support golden', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.txt': `Hello world`,
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Hello world').toMatchSnapshot('snapshot.txt');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should fail on wrong golden', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.txt': `Line1
+Line2
+Line3
+Hello world line1
+Line5
+Line6
+Line7`,
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        const data = [];
+        data.push('Line1');
+        data.push('Line22');
+        data.push('Line3');
+        data.push('Hi world line2');
+        data.push('Line5');
+        data.push('Line6');
+        data.push('Line7');
+        expect(data.join('\\n')).toMatchSnapshot('snapshot.txt');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Line1');
+  expect(result.output).toContain('Line2' + colors.green('2'));
+  expect(result.output).toContain('line' + colors.strikethrough(colors.red('1')) + colors.green('2'));
+  expect(result.output).toContain('Line3');
+  expect(result.output).toContain('Line5');
+  expect(result.output).toContain('Line7');
+});
+
+test('should write missing expectations locally', async ({runInlineTest}, testInfo) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Hello world').toMatchSnapshot('snapshot.txt');
+      });
+    `
+  }, {}, { CI: '' });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('snapshot.txt is missing in snapshots, writing actual');
+  const data = fs.readFileSync(testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'));
+  expect(data.toString()).toBe('Hello world');
+});
+
+test('should not write missing expectations on CI', async ({runInlineTest}, testInfo) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Hello world').toMatchSnapshot('snapshot.txt');
+      });
+    `
+  }, {}, { CI: '1' });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('snapshot.txt is missing in snapshots');
+  expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'))).toBe(false);
+});
+
+test('should update expectations', async ({runInlineTest}, testInfo) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.txt': `Hello world`,
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Hello world updated').toMatchSnapshot('snapshot.txt');
+      });
+    `
+  }, { 'update-snapshots': true });
+  expect(result.exitCode).toBe(0);
+  expect(result.output).toContain('snapshot.txt does not match, writing actual.');
+  const data = fs.readFileSync(testInfo.outputPath('a.spec.js-snapshots/snapshot.txt'));
+  expect(data.toString()).toBe('Hello world updated');
+});
+
+test('should match multiple snapshots', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot1.txt': `Snapshot1`,
+    'a.spec.js-snapshots/snapshot2.txt': `Snapshot2`,
+    'a.spec.js-snapshots/snapshot3.txt': `Snapshot3`,
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Snapshot1').toMatchSnapshot('snapshot1.txt');
+        expect('Snapshot2').toMatchSnapshot('snapshot2.txt');
+        expect('Snapshot3').toMatchSnapshot('snapshot3.txt');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should match snapshots from multiple projects', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = { projects: [
+        { testDir: path.join(__dirname, 'p1') },
+        { testDir: path.join(__dirname, 'p2') },
+      ]};
+    `,
+    'p1/a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Snapshot1').toMatchSnapshot('snapshot.txt');
+      });
+    `,
+    'p1/a.spec.js-snapshots/snapshot.txt': `Snapshot1`,
+    'p2/a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Snapshot2').toMatchSnapshot('snapshot.txt');
+      });
+    `,
+    'p2/a.spec.js-snapshots/snapshot.txt': `Snapshot2`,
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should use provided name', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/provided.txt': `Hello world`,
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Hello world').toMatchSnapshot('provided.txt');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should throw without a name', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Hello world').toMatchSnapshot();
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('toMatchSnapshot() requires a "name" parameter');
+});
+
+test('should use provided name via options', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/provided.txt': `Hello world`,
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect('Hello world').toMatchSnapshot({ name: 'provided.txt' });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should compare binary', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.dat': Buffer.from([1,2,3,4]),
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect(Buffer.from([1,2,3,4])).toMatchSnapshot('snapshot.dat');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should compare PNG images', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.png':
+        Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'),
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64')).toMatchSnapshot('snapshot.png');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should compare different PNG images', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.png':
+        Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'),
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII==', 'base64')).toMatchSnapshot('snapshot.png');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Snapshot comparison failed');
+  expect(result.output).toContain('snapshot-diff.png');
+});
+
+test('should respect threshold', async ({runInlineTest}) => {
+  const expected = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-expected.png'));
+  const actual = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-actual.png'));
+  const result = await runInlineTest({
+    'a.spec.js-snapshots/snapshot.png': expected,
+    'a.spec.js-snapshots/snapshot2.png': expected,
+    'a.spec.js': `
+      const { test } = folio;
+      test('is a test', ({}) => {
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { threshold: 0.3 });
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).not.toMatchSnapshot('snapshot.png', { threshold: 0.2 });
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot2.png', { threshold: 0.3 });
+        expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot({ name: 'snapshot2.png', threshold: 0.3 });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/hooks.spec.ts

@@ -0,0 +1,192 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('hooks should work with fixtures', async ({ runInlineTest }) => {
+  const { results } = await runInlineTest({
+    'helper.ts': `
+      global.logs = [];
+      export const test = folio.test.extend({
+        w: [ async ({}, run) => {
+          global.logs.push('+w');
+          await run(17);
+          global.logs.push('-w');
+        }, { scope: 'worker' }],
+
+        t: async ({}, run) => {
+          global.logs.push('+t');
+          await run(42);
+          global.logs.push('-t');
+        },
+      });
+    `,
+    'a.test.js': `
+      const { test } = require('./helper');
+      test.describe('suite', () => {
+        test.beforeAll(async ({ w }) => {
+          global.logs.push('beforeAll-' + w);
+        });
+        test.afterAll(async ({ w }) => {
+          global.logs.push('afterAll-' + w);
+        });
+
+        test.beforeEach(async ({t}) => {
+          global.logs.push('beforeEach-' + t);
+        });
+        test.afterEach(async ({t}) => {
+          global.logs.push('afterEach-' + t);
+        });
+
+        test('one', async ({t}) => {
+          global.logs.push('test');
+          expect(t).toBe(42);
+        });
+      });
+
+      test('two', async ({t}) => {
+        expect(global.logs).toEqual([
+          '+w',
+          'beforeAll-17',
+          '+t',
+          'beforeEach-42',
+          'test',
+          'afterEach-42',
+          '-t',
+          'afterAll-17',
+          '+t',
+        ]);
+      });
+    `,
+  });
+  expect(results[0].status).toBe('passed');
+});
+
+test('afterEach failure should not prevent other hooks and fixtures teardown', async ({ runInlineTest }) => {
+  const report = await runInlineTest({
+    'helper.ts': `
+      global.logs = [];
+      export const test = folio.test.extend({
+        foo: async ({}, run) => {
+          console.log('+t');
+          await run();
+          console.log('-t');
+        }
+      });
+    `,
+    'a.test.js': `
+      const { test } = require('./helper');
+      test.describe('suite', () => {
+        test.afterEach(async () => {
+          console.log('afterEach1');
+        });
+        test.afterEach(async () => {
+          console.log('afterEach2');
+          throw new Error('afterEach2');
+        });
+        test('one', async ({foo}) => {
+          console.log('test');
+          expect(true).toBe(true);
+        });
+      });
+    `,
+  });
+  expect(report.output).toContain('+t\ntest\nafterEach2\nafterEach1\n-t');
+  expect(report.results[0].error.message).toContain('afterEach2');
+});
+
+test('beforeEach failure should prevent the test, but not other hooks', async ({ runInlineTest }) => {
+  const report = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test.describe('suite', () => {
+        test.beforeEach(async ({}) => {
+          console.log('beforeEach1');
+        });
+        test.beforeEach(async ({}) => {
+          console.log('beforeEach2');
+          throw new Error('beforeEach2');
+        });
+        test.afterEach(async ({}) => {
+          console.log('afterEach');
+        });
+        test('one', async ({}) => {
+          console.log('test');
+        });
+      });
+    `,
+  });
+  expect(report.output).toContain('beforeEach1\nbeforeEach2\nafterEach');
+  expect(report.results[0].error.message).toContain('beforeEach2');
+});
+
+test('beforeAll should be run once', async ({ runInlineTest }) => {
+  const report = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test.describe('suite1', () => {
+        let counter = 0;
+        test.beforeAll(async () => {
+          console.log('beforeAll1-' + (++counter));
+        });
+        test.describe('suite2', () => {
+          test.beforeAll(async () => {
+            console.log('beforeAll2');
+          });
+          test('one', async ({}) => {
+            console.log('test');
+          });
+        });
+      });
+    `,
+  });
+  expect(report.output).toContain('beforeAll1-1\nbeforeAll2\ntest');
+});
+
+test('beforeEach should be able to skip a test', async ({ runInlineTest }) => {
+  const { passed, skipped, exitCode } = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test.beforeEach(async ({}, testInfo) => {
+        testInfo.skip(testInfo.title === 'test2');
+      });
+      test('test1', async () => {});
+      test('test2', async () => {});
+    `,
+  });
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(1);
+  expect(skipped).toBe(1);
+});
+
+test('beforeAll from a helper file should throw', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'my-test.ts': `
+      export const test = folio.test;
+      test.beforeAll(() => {});
+    `,
+    'playwright.config.ts': `
+      import { test } from './my-test';
+    `,
+    'a.test.ts': `
+      import { test } from './my-test';
+      test('should work', async () => {
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('beforeAll hook can only be called in a test file');
+});

tests/playwright-test/json-reporter.spec.ts

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as path from 'path';
+import { test, expect } from './playwright-test-fixtures';
+
+test('should support spec.ok', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('math works!', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+      test('math fails!', async ({}) => {
+        expect(1 + 1).toBe(3);
+      });
+    `
+  }, { });
+  expect(result.exitCode).toBe(1);
+  expect(result.report.suites[0].specs[0].ok).toBe(true);
+  expect(result.report.suites[0].specs[1].ok).toBe(false);
+});
+
+test('should report projects', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        retries: 2,
+        projects: [
+          {
+            timeout: 5000,
+            name: 'p1',
+            metadata: { foo: 'bar' },
+          },
+          {
+            timeout: 8000,
+            name: 'p2',
+            metadata: { bar: 42 },
+          }
+        ]
+      };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('math works!', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { });
+  expect(result.exitCode).toBe(0);
+  const projects = result.report.config.projects;
+  const testDir = testInfo.outputDir.split(path.sep).join(path.posix.sep);
+
+  expect(projects[0].name).toBe('p1');
+  expect(projects[0].retries).toBe(2);
+  expect(projects[0].timeout).toBe(5000);
+  expect(projects[0].metadata).toEqual({ foo: 'bar' });
+  expect(projects[0].testDir).toBe(testDir);
+
+  expect(projects[1].name).toBe('p2');
+  expect(projects[1].retries).toBe(2);
+  expect(projects[1].timeout).toBe(8000);
+  expect(projects[1].metadata).toEqual({ bar: 42 });
+  expect(projects[1].testDir).toBe(testDir);
+
+  expect(result.report.suites[0].specs[0].tests[0].projectName).toBe('p1');
+  expect(result.report.suites[0].specs[0].tests[1].projectName).toBe('p2');
+});

tests/playwright-test/junit-reporter.spec.ts

@@ -0,0 +1,193 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import xml2js from 'xml2js';
+import { test, expect } from './playwright-test-fixtures';
+
+test('should render expected', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}) => {
+        expect(1).toBe(1);
+      });
+    `,
+    'b.test.js': `
+      const { test } = folio;
+      test('two', async ({}) => {
+        expect(1).toBe(1);
+      });
+    `,
+  }, { reporter: 'junit' });
+  const xml = parseXML(result.output);
+  expect(xml['testsuites']['$']['tests']).toBe('2');
+  expect(xml['testsuites']['$']['failures']).toBe('0');
+  expect(xml['testsuites']['testsuite'].length).toBe(2);
+  expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
+  expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
+  expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
+  expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
+  expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('b.test.js');
+  expect(result.exitCode).toBe(0);
+});
+
+test('should render unexpected', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}) => {
+        expect(1).toBe(0);
+      });
+    `,
+  }, { reporter: 'junit' });
+  const xml = parseXML(result.output);
+  expect(xml['testsuites']['$']['tests']).toBe('1');
+  expect(xml['testsuites']['$']['failures']).toBe('1');
+  const failure = xml['testsuites']['testsuite'][0]['testcase'][0]['failure'][0];
+  expect(failure['$']['message']).toContain('a.test.js');
+  expect(failure['$']['message']).toContain('one');
+  expect(failure['$']['type']).toBe('FAILURE');
+  expect(failure['_']).toContain('expect(1).toBe(0)');
+  expect(result.exitCode).toBe(1);
+});
+
+test('should render unexpected after retry', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}) => {
+        expect(1).toBe(0);
+      });
+    `,
+  }, { retries: 3, reporter: 'junit' });
+  expect(result.output).toContain(`tests="1"`);
+  expect(result.output).toContain(`failures="1"`);
+  expect(result.output).toContain(`<failure`);
+  expect(result.output).toContain('Retry #1');
+  expect(result.output).toContain('Retry #2');
+  expect(result.output).toContain('Retry #3');
+  expect(result.exitCode).toBe(1);
+});
+
+test('should render flaky', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(3);
+      });
+    `,
+  }, { retries: 3, reporter: 'junit' });
+  expect(result.output).not.toContain('Retry #1');
+  expect(result.exitCode).toBe(0);
+});
+
+test('should render stdout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      import colors from 'colors/safe';
+      const { test } = folio;
+      test('one', async ({}) => {
+        console.log(colors.yellow('Hello world'));
+        test.expect("abc").toBe('abcd');
+      });
+    `,
+  }, { reporter: 'junit' });
+  const xml = parseXML(result.output);
+  const suite = xml['testsuites']['testsuite'][0];
+  expect(suite['system-out'].length).toBe(1);
+  expect(suite['system-out'][0]).toContain('Hello world');
+  expect(suite['system-out'][0]).not.toContain('u00');
+  expect(suite['testcase'][0]['failure'][0]['_']).toContain(`>  9 |         test.expect("abc").toBe('abcd');`);
+  expect(result.exitCode).toBe(1);
+});
+
+test('should render stdout without ansi escapes', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: [ ['junit', { stripANSIControlSequences: true }] ],
+      };
+    `,
+    'a.test.ts': `
+      import colors from 'colors/safe';
+      const { test } = folio;
+      test('one', async ({}) => {
+        console.log(colors.yellow('Hello world'));
+      });
+    `,
+  }, { reporter: '' });
+  const xml = parseXML(result.output);
+  const suite = xml['testsuites']['testsuite'][0];
+  expect(suite['system-out'].length).toBe(1);
+  expect(suite['system-out'][0].trim()).toBe('Hello world');
+  expect(result.exitCode).toBe(0);
+});
+
+test('should render skipped', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async () => {
+        console.log('Hello world');
+      });
+      test('two', async () => {
+        test.skip();
+        console.log('Hello world');
+      });
+    `,
+  }, { retries: 3, reporter: 'junit' });
+  const xml = parseXML(result.output);
+  expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
+  expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
+  expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1');
+  expect(result.exitCode).toBe(0);
+});
+
+test('should render projects', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [ { name: 'project1' }, { name: 'project2' } ] };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}) => {
+        expect(1).toBe(1);
+      });
+    `,
+  }, { reporter: 'junit' });
+  const xml = parseXML(result.output);
+  expect(xml['testsuites']['$']['tests']).toBe('2');
+  expect(xml['testsuites']['$']['failures']).toBe('0');
+  expect(xml['testsuites']['testsuite'].length).toBe(1);
+  expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
+  expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
+  expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
+  expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
+  expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
+  expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toContain('[project1] one');
+  expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toContain('a.test.js:6:7');
+  expect(xml['testsuites']['testsuite'][0]['testcase'][1]['$']['name']).toBe('one');
+  expect(xml['testsuites']['testsuite'][0]['testcase'][1]['$']['classname']).toContain('[project2] one');
+  expect(xml['testsuites']['testsuite'][0]['testcase'][1]['$']['classname']).toContain('a.test.js:6:7');
+  expect(result.exitCode).toBe(0);
+});
+
+function parseXML(xml: string): any {
+  let result: any;
+  xml2js.parseString(xml, (err, r) => result = r);
+  return result;
+}

tests/playwright-test/line-reporter.spec.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+
+test('render unexpected after retry', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}) => {
+        expect(1).toBe(0);
+      });
+    `,
+  }, { retries: 3, reporter: 'line' });
+  const text = stripAscii(result.output);
+  expect(text).toContain('1 failed');
+  expect(text).toContain('1) a.test');
+  expect(text).not.toContain('2) a.test');
+  expect(text).toContain('Retry #1');
+  expect(text).toContain('Retry #2');
+  expect(text).toContain('Retry #3');
+  expect(result.exitCode).toBe(1);
+});
+
+test('render flaky', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('one', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(3);
+      });
+    `,
+  }, { retries: 3, reporter: 'line' });
+  const text = stripAscii(result.output);
+  expect(text).toContain('1 flaky');
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/list-mode.spec.ts

@@ -0,0 +1,34 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import { test, expect } from './playwright-test-fixtures';
+
+test('should have relative always-posix paths', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('math works!', async ({}) => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { 'list': true });
+  expect(result.exitCode).toBe(0);
+  expect(result.report.config.rootDir.indexOf(path.win32.sep)).toBe(-1);
+  expect(result.report.suites[0].specs[0].file).toBe('a.test.js');
+  expect(result.report.suites[0].specs[0].line).toBe(6);
+  expect(result.report.suites[0].specs[0].column).toBe(7);
+});

tests/playwright-test/list-reporter.spec.ts

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+
+test('render each test with project name', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'foo' },
+        { name: 'bar' },
+      ] };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('fails', async ({}) => {
+        expect(1).toBe(0);
+      });
+      test('passes', async ({}) => {
+        expect(0).toBe(0);
+      });
+    `,
+  }, { reporter: 'list' });
+  const text = stripAscii(result.output);
+  expect(text).toContain('a.test.ts:6:7 › [foo] fails');
+  expect(text).toContain('a.test.ts:6:7 › [bar] fails');
+  expect(text).toContain('a.test.ts:9:7 › [foo] passes');
+  expect(text).toContain('a.test.ts:9:7 › [bar] passes');
+  expect(result.exitCode).toBe(1);
+});

tests/playwright-test/match-grep.spec.ts

@@ -0,0 +1,98 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+const files = {
+  'match-grep/b.test.ts': `
+    const { test } = folio;
+    test('test AA', () => {
+      expect(1 + 1).toBe(2);
+    });
+
+    test('test BB', () => {
+      expect(1 + 1).toBe(2);
+    });
+
+    test('test CC', () => {
+      expect(1 + 1).toBe(2);
+    });
+  `,
+  'match-grep/fdir/c.test.ts': `
+    const { test } = folio;
+    test('test AA', () => {
+      expect(1 + 1).toBe(2);
+    });
+
+    test('test BB', () => {
+      expect(1 + 1).toBe(2);
+    });
+
+    test('test CC', () => {
+      expect(1 + 1).toBe(2);
+    });
+  `,
+  'match-grep/adir/a.test.ts': `
+    const { test } = folio;
+    test('test AA', () => {
+      expect(1 + 1).toBe(2);
+    });
+
+    test('test BB', () => {
+      expect(1 + 1).toBe(2);
+    });
+
+    test('test CC', () => {
+      expect(1 + 1).toBe(2);
+    });
+  `,
+};
+
+test('should grep test name', async ({ runInlineTest }) => {
+  const result = await runInlineTest(files, { 'grep': 'test [A-B]' });
+  expect(result.passed).toBe(6);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should grep test name with //', async ({ runInlineTest }) => {
+  const result = await runInlineTest(files, { 'grep': '/B$/' });
+  expect(result.passed).toBe(3);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should grep test name with //', async ({ runInlineTest }) => {
+  const result = await runInlineTest(files, { 'grep': '/TesT c/i' });
+  expect(result.passed).toBe(3);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should grep by project name', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'foo' },
+        { name: 'bar' },
+      ]};
+    `,
+    'a.spec.ts': `
+      folio.test('should work', () => {});
+    `,
+  }, { 'grep': 'foo]' });
+  expect(result.passed).toBe(1);
+  expect(result.skipped).toBe(0);
+  expect(result.failed).toBe(0);
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/max-failures.spec.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('max-failures should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = folio;
+      for (let i = 0; i < 10; ++i) {
+        test('fail_' + i, () => {
+          expect(true).toBe(false);
+        });
+      }
+    `,
+    'b.spec.js': `
+      const { test } = folio;
+      for (let i = 0; i < 10; ++i) {
+        test('fail_' + i, () => {
+          expect(true).toBe(false);
+        });
+      }
+    `
+  }, { 'max-failures': 8 });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(8);
+  expect(result.output.split('\n').filter(l => l.includes('expect(')).length).toBe(16);
+});
+
+test('-x should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = folio;
+      for (let i = 0; i < 10; ++i) {
+        test('fail_' + i, () => {
+          expect(true).toBe(false);
+        });
+      }
+    `,
+    'b.spec.js': `
+      const { test } = folio;
+      for (let i = 0; i < 10; ++i) {
+        test('fail_' + i, () => {
+          expect(true).toBe(false);
+        });
+      }
+    `
+  }, { '-x': true });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output.split('\n').filter(l => l.includes('expect(')).length).toBe(2);
+});

tests/playwright-test/options.spec.ts

@@ -0,0 +1,166 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should merge options', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const test = folio.test.extend({
+        foo: 'foo',
+        bar: 'bar',
+      });
+
+      test.use({ foo: 'foo2' });
+      test.use({ bar: 'bar2' });
+      test('test', ({ foo, bar }) => {
+        expect(foo).toBe('foo2');
+        expect(bar).toBe('bar2');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should run tests with different test options in the same worker', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      export const test = folio.test.extend({
+        foo: 'foo',
+      });
+    `,
+    'a.test.ts': `
+      import { test } from './helper';
+      test('test', ({ foo }, testInfo) => {
+        expect(foo).toBe('foo');
+        expect(testInfo.workerIndex).toBe(0);
+      });
+
+      test.describe('suite1', () => {
+        test.use({ foo: 'bar' });
+        test('test1', ({ foo }, testInfo) => {
+          expect(foo).toBe('bar');
+          expect(testInfo.workerIndex).toBe(0);
+        });
+
+        test.describe('suite2', () => {
+          test.use({ foo: 'baz' });
+          test('test2', ({ foo }, testInfo) => {
+            expect(foo).toBe('baz');
+            expect(testInfo.workerIndex).toBe(0);
+          });
+        });
+      });
+    `
+  }, { workers: 1 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+});
+
+test('should run tests with different worker options', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      export const test = folio.test.extend({
+        foo: [undefined, { scope: 'worker' }],
+      });
+    `,
+    'a.test.ts': `
+      import { test } from './helper';
+      test('test', ({ foo }, testInfo) => {
+        expect(foo).toBe(undefined);
+        console.log('\\n%%test=' + testInfo.workerIndex);
+      });
+
+      test.describe('suite1', () => {
+        test.use({ foo: 'bar' });
+        test('test1', ({ foo }, testInfo) => {
+          expect(foo).toBe('bar');
+          console.log('\\n%%test1=' + testInfo.workerIndex);
+        });
+
+        test.describe('suite2', () => {
+          test.use({ foo: 'baz' });
+          test('test2', ({ foo }, testInfo) => {
+            expect(foo).toBe('baz');
+            console.log('\\n%%test2=' + testInfo.workerIndex);
+          });
+        });
+
+        test('test3', ({ foo }, testInfo) => {
+          expect(foo).toBe('bar');
+          console.log('\\n%%test3=' + testInfo.workerIndex);
+        });
+      });
+    `,
+    'b.test.ts': `
+      import { test } from './helper';
+      test.use({ foo: 'qux' });
+      test('test4', ({ foo }, testInfo) => {
+        expect(foo).toBe('qux');
+        console.log('\\n%%test4=' + testInfo.workerIndex);
+      });
+    `
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(5);
+
+  const workerIndexMap = new Map();
+  const allWorkers = new Set();
+  for (const line of result.output.split('\n')) {
+    if (line.startsWith('%%')) {
+      const [ name, workerIndex ] = line.substring(2).split('=');
+      allWorkers.add(workerIndex);
+      workerIndexMap.set(name, workerIndex);
+    }
+  }
+
+  expect(workerIndexMap.size).toBe(5);
+  expect(workerIndexMap.get('test1')).toBe(workerIndexMap.get('test3'));
+  expect(allWorkers.size).toBe(4);
+  for (let i = 0; i < 4; i++)
+    expect(allWorkers.has(String(i)));
+});
+
+test('should use options from the config', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      export const test = folio.test.extend({
+        foo: 'foo',
+      });
+    `,
+    'playwright.config.ts': `
+      module.exports = { use: { foo: 'bar' } };
+    `,
+    'a.test.ts': `
+      import { test } from './helper';
+      test('test1', ({ foo }) => {
+        expect(foo).toBe('bar');
+      });
+
+      test.describe('suite1', () => {
+        test.use({ foo: 'baz' });
+
+        test('test2', ({ foo }) => {
+          expect(foo).toBe('baz');
+        });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+});

tests/playwright-test/override-timeout.spec.ts

@@ -0,0 +1,70 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should consider dynamically set value', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = { timeout: 100 };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('pass', ({}, testInfo) => {
+        expect(testInfo.timeout).toBe(100);
+      })
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should allow different timeouts', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = { projects: [
+        { timeout: 200 },
+        { timeout: 100 },
+      ] };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('pass', ({}, testInfo) => {
+        console.log('timeout:' + testInfo.timeout);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+  expect(result.output).toContain('timeout:100');
+  expect(result.output).toContain('timeout:200');
+});
+
+test('should prioritize value set via command line', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = { timeout: 100 };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('pass', ({}, testInfo) => {
+        expect(testInfo.timeout).toBe(1000);
+      })
+    `
+  }, { timeout: 1000 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});

tests/playwright-test/playwright-test-fixtures.ts

@@ -0,0 +1,262 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TestInfo, test as base } from '../config/test-runner';
+import { spawn } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import type { ReportFormat } from '../../src/test/reporters/json';
+import rimraf from 'rimraf';
+import { promisify } from 'util';
+
+const removeFolderAsync = promisify(rimraf);
+
+type RunResult = {
+  exitCode: number,
+  output: string,
+  passed: number,
+  failed: number,
+  flaky: number,
+  skipped: number,
+  report: ReportFormat,
+  results: any[],
+};
+
+type TSCResult = {
+  output: string;
+  exitCode: number;
+};
+
+type Files = { [key: string]: string | Buffer };
+type Params = { [key: string]: string | number | boolean | string[] };
+type Env = { [key: string]: string | number | boolean | undefined };
+
+async function writeFiles(testInfo: TestInfo, files: Files) {
+  const baseDir = testInfo.outputPath();
+
+  const internalPath = JSON.stringify(path.join(__dirname, 'playwright-test-internal'));
+  const headerJS = `
+    const folio = require(${internalPath});
+  `;
+  const headerTS = `
+    import * as folio from ${internalPath};
+  `;
+
+  const hasConfig = Object.keys(files).some(name => name.includes('.config.'));
+  if (!hasConfig) {
+    files = {
+      ...files,
+      'playwright.config.ts': `
+        module.exports = { projects: [ {} ] };
+      `,
+    };
+  }
+
+  await Promise.all(Object.keys(files).map(async name => {
+    const fullName = path.join(baseDir, name);
+    await fs.promises.mkdir(path.dirname(fullName), { recursive: true });
+    const isTypeScriptSourceFile = name.endsWith('ts') && !name.endsWith('d.ts');
+    const header = isTypeScriptSourceFile ? headerTS : headerJS;
+    if (/(spec|test)\.(js|ts)$/.test(name)) {
+      const fileHeader = header + 'const { expect } = folio;\n';
+      await fs.promises.writeFile(fullName, fileHeader + files[name]);
+    } else if (/\.(js|ts)$/.test(name) && !name.endsWith('d.ts')) {
+      await fs.promises.writeFile(fullName, header + files[name]);
+    } else {
+      await fs.promises.writeFile(fullName, files[name]);
+    }
+  }));
+
+  return baseDir;
+}
+
+async function runTSC(baseDir: string): Promise<TSCResult> {
+  const tscProcess = spawn('npx', ['tsc', '-p', baseDir], {
+    cwd: baseDir,
+    shell: true,
+  });
+  let output = '';
+  tscProcess.stderr.on('data', chunk => {
+    output += String(chunk);
+    if (process.env.PW_RUNNER_DEBUG)
+      process.stderr.write(String(chunk));
+  });
+  tscProcess.stdout.on('data', chunk => {
+    output += String(chunk);
+    if (process.env.PW_RUNNER_DEBUG)
+      process.stdout.write(String(chunk));
+  });
+  const status = await new Promise<number>(x => tscProcess.on('close', x));
+  return {
+    exitCode: status,
+    output,
+  };
+}
+
+async function runFolio(baseDir: string, params: any, env: Env): Promise<RunResult> {
+  const paramList = [];
+  let additionalArgs = '';
+  for (const key of Object.keys(params)) {
+    if (key === 'args') {
+      additionalArgs = params[key];
+      continue;
+    }
+    for (const value of Array.isArray(params[key]) ? params[key] : [params[key]]) {
+      const k = key.startsWith('-') ? key : '--' + key;
+      paramList.push(params[key] === true ? `${k}` : `${k}=${value}`);
+    }
+  }
+  const outputDir = path.join(baseDir, 'test-results');
+  const reportFile = path.join(outputDir, 'report.json');
+  const args = [path.join(__dirname, '..', '..', 'lib', 'cli', 'cli.js'), 'test'];
+  args.push(
+      '--output=' + outputDir,
+      '--reporter=dot,json',
+      '--workers=2',
+      ...paramList
+  );
+  if (additionalArgs)
+    args.push(...additionalArgs);
+  const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'playwright-test-cache-'));
+  const testProcess = spawn('node', args, {
+    env: {
+      ...process.env,
+      ...env,
+      PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile,
+      PWTEST_CACHE_DIR: cacheDir,
+      PWTEST_CLI_ALLOW_TEST_COMMAND: '1',
+    },
+    cwd: baseDir
+  });
+  let output = '';
+  testProcess.stderr.on('data', chunk => {
+    output += String(chunk);
+    if (process.env.PW_RUNNER_DEBUG)
+      process.stderr.write(String(chunk));
+  });
+  testProcess.stdout.on('data', chunk => {
+    output += String(chunk);
+    if (process.env.PW_RUNNER_DEBUG)
+      process.stdout.write(String(chunk));
+  });
+  const status = await new Promise<number>(x => testProcess.on('close', x));
+  await removeFolderAsync(cacheDir);
+
+  const outputString = output.toString();
+  const summary = (re: RegExp) => {
+    let result = 0;
+    let match = re.exec(outputString);
+    while (match) {
+      result += (+match[1]);
+      match = re.exec(outputString);
+    }
+    return result;
+  };
+  const passed = summary(/(\d+) passed/g);
+  const failed = summary(/(\d+) failed/g);
+  const flaky = summary(/(\d+) flaky/g);
+  const skipped = summary(/(\d+) skipped/g);
+  let report;
+  try {
+    report = JSON.parse(fs.readFileSync(reportFile).toString());
+  } catch (e) {
+    output += '\n' + e.toString();
+  }
+
+  const results = [];
+  function visitSuites(suites?: ReportFormat['suites']) {
+    if (!suites)
+      return;
+    for (const suite of suites) {
+      for (const spec of suite.specs) {
+        for (const test of spec.tests)
+          results.push(...test.results);
+      }
+      visitSuites(suite.suites);
+    }
+  }
+  if (report)
+    visitSuites(report.suites);
+
+  return {
+    exitCode: status,
+    output,
+    passed,
+    failed,
+    flaky,
+    skipped,
+    report,
+    results,
+  };
+}
+
+type Fixtures = {
+  writeFiles: (files: Files) => Promise<string>;
+  runInlineTest: (files: Files, params?: Params, env?: Env) => Promise<RunResult>;
+  runTSC: (files: Files) => Promise<TSCResult>;
+};
+
+export const test = base.extend<Fixtures>({
+  writeFiles: async ({}, use, testInfo) => {
+    await use(files => writeFiles(testInfo, files));
+  },
+
+  runInlineTest: async ({}, use, testInfo: TestInfo) => {
+    let runResult: RunResult | undefined;
+    await use(async (files: Files, params: Params = {}, env: Env = {}) => {
+      const baseDir = await writeFiles(testInfo, files);
+      runResult = await runFolio(baseDir, params, env);
+      return runResult;
+    });
+    if (testInfo.status !== testInfo.expectedStatus && runResult)
+      console.log(runResult.output);
+  },
+
+  runTSC: async ({}, use, testInfo) => {
+    let tscResult: TSCResult | undefined;
+    await use(async files => {
+      const baseDir = await writeFiles(testInfo, { ...files, 'tsconfig.json': JSON.stringify(TSCONFIG) });
+      tscResult = await runTSC(baseDir);
+      return tscResult;
+    });
+    if (testInfo.status !== testInfo.expectedStatus && tscResult)
+      console.log(tscResult.output);
+  },
+});
+
+const TSCONFIG = {
+  'compilerOptions': {
+    'target': 'ESNext',
+    'moduleResolution': 'node',
+    'module': 'commonjs',
+    'strict': true,
+    'esModuleInterop': true,
+    'allowSyntheticDefaultImports': true,
+    'rootDir': '.',
+    'lib': ['esnext', 'dom', 'DOM.Iterable']
+  },
+  'exclude': [
+    'node_modules'
+  ]
+};
+
+export { expect } from '../config/test-runner';
+
+const asciiRegex = new RegExp('[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))', 'g');
+export function stripAscii(str: string): string {
+  return str.replace(asciiRegex, '');
+}

tests/playwright-test/playwright-test-internal.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { TestType } from '../../types/testInternal';
+import type { Expect } from '../../types/testExpect';
+export type { Project, Config, TestStatus, TestInfo, WorkerInfo, TestType, Fixtures, TestFixture, WorkerFixture } from '../../types/testInternal';
+export const test: TestType<{}, {}>;
+export default test;
+export const expect: Expect;

tests/playwright-test/playwright-test-internal.js

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module.exports = require('../../lib/test/internal');

tests/playwright-test/playwright-test.config.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '../config/test-runner';
+
+const config: Config = {
+  testDir: __dirname,
+  testIgnore: 'assets/**',
+  timeout: 20000,
+  forbidOnly: !!process.env.CI,
+  projects: [
+    { name: 'playwright-test' },
+  ]
+};
+
+export default config;

tests/playwright-test/repeat-each.spec.ts

@@ -0,0 +1,54 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should repeat from command line', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = folio;
+      test('test', ({}, testInfo) => {
+        console.log('REPEAT ' + testInfo.repeatEachIndex);
+        expect(1).toBe(1);
+      });
+    `
+  }, { 'repeat-each': 3 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  expect(result.output).toContain('REPEAT 0');
+  expect(result.output).toContain('REPEAT 1');
+  expect(result.output).toContain('REPEAT 2');
+  expect(result.output).not.toContain('REPEAT 3');
+});
+
+test('should repeat based on config', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = { projects: [
+        { name: 'no-repeats' },
+        { repeatEach: 2, name: 'two-repeats' },
+      ] };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('my test', ({}, testInfo) => {});
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  const names = result.report.suites[0].specs[0].tests.map(test => test.projectName);
+  expect(names).toEqual(['no-repeats', 'two-repeats', 'two-repeats']);
+});

tests/playwright-test/retry.spec.ts

@@ -0,0 +1,186 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAscii } from './playwright-test-fixtures';
+
+test('should retry failures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'retry-failures.spec.js': `
+      const { test } = folio;
+      test('flake', async ({}, testInfo) => {
+        // Passes on the second run.
+        expect(testInfo.retry).toBe(1);
+      });
+    `
+  }, { retries: 10 });
+  expect(result.exitCode).toBe(0);
+  expect(result.flaky).toBe(1);
+  expect(result.results.length).toBe(2);
+  expect(result.results[0].workerIndex).toBe(0);
+  expect(result.results[0].retry).toBe(0);
+  expect(result.results[0].status).toBe('failed');
+  expect(result.results[1].workerIndex).toBe(1);
+  expect(result.results[1].retry).toBe(1);
+  expect(result.results[1].status).toBe('passed');
+});
+
+test('should retry based on config', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = { projects: [
+        { retries: 0, name: 'no-retries' },
+        { retries: 2, name: 'two-retries' },
+      ] };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('pass', ({}, testInfo) => {
+        // Passes on the third run.
+        expect(testInfo.retry).toBe(2);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.flaky).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.results.length).toBe(4);
+});
+
+test('should retry timeout', async ({ runInlineTest }) => {
+  const { exitCode, passed, failed, output } = await runInlineTest({
+    'one-timeout.spec.js': `
+      const { test } = folio;
+      test('timeout', async () => {
+        await new Promise(f => setTimeout(f, 10000));
+      });
+    `
+  }, { timeout: 100, retries: 2 });
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(0);
+  expect(failed).toBe(1);
+  expect(stripAscii(output).split('\n')[0]).toBe('××T');
+});
+
+test('should fail on unexpected pass with retries', async ({ runInlineTest }) => {
+  const { exitCode, failed, output } = await runInlineTest({
+    'unexpected-pass.spec.js': `
+      const { test } = folio;
+      test('succeeds', () => {
+        test.fail();
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { retries: 1 });
+  expect(exitCode).toBe(1);
+  expect(failed).toBe(1);
+  expect(output).toContain('passed unexpectedly');
+});
+
+test('should not retry unexpected pass', async ({ runInlineTest }) => {
+  const { exitCode, passed, failed, output } = await runInlineTest({
+    'unexpected-pass.spec.js': `
+      const { test } = folio;
+      test('succeeds', () => {
+        test.fail();
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { retries: 2 });
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(0);
+  expect(failed).toBe(1);
+  expect(stripAscii(output).split('\n')[0]).toBe('F');
+});
+
+test('should not retry expected failure', async ({ runInlineTest }) => {
+  const { exitCode, passed, failed, output } = await runInlineTest({
+    'expected-failure.spec.js': `
+      const { test } = folio;
+      test('fails', () => {
+        test.fail();
+        expect(1 + 1).toBe(3);
+      });
+
+      test('non-empty remaining',() => {
+        expect(1 + 1).toBe(2);
+      });
+    `
+  }, { retries: 2 });
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(2);
+  expect(failed).toBe(0);
+  expect(stripAscii(output).split('\n')[0]).toBe('··');
+});
+
+test('should retry unhandled rejection', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'unhandled-rejection.spec.js': `
+      const { test } = folio;
+      test('unhandled rejection', async () => {
+        setTimeout(() => {
+          throw new Error('Unhandled rejection in the test');
+        });
+        await new Promise(f => setTimeout(f, 20));
+      });
+    `
+  }, { retries: 2 });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.failed).toBe(1);
+  expect(stripAscii(result.output).split('\n')[0]).toBe('××F');
+  expect(result.output).toContain('Unhandled rejection');
+});
+
+test('should retry beforeAll failure', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = folio;
+      test.beforeAll(async () => {
+        throw new Error('BeforeAll is bugged!');
+      });
+      test('passing test', async () => {
+      });
+    `
+  }, { retries: 2 });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.failed).toBe(1);
+  expect(stripAscii(result.output).split('\n')[0]).toBe('××F');
+  expect(result.output).toContain('BeforeAll is bugged!');
+});
+
+test('should retry worker fixture setup failure', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      export const test = folio.test.extend({
+        worker: [ async () => {
+          throw new Error('worker setup is bugged!');
+        }, { scope: 'worker' } ]
+      });
+    `,
+    'a.spec.ts': `
+      import { test } from './helper';
+      test('passing test', async ({ worker }) => {
+      });
+    `
+  }, { retries: 2 });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.failed).toBe(1);
+  expect(stripAscii(result.output).split('\n')[0]).toBe('××F');
+  expect(result.output).toContain('worker setup is bugged!');
+});

tests/playwright-test/shard.spec.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+const tests = {
+  'a.spec.ts': `
+      const { test } = folio;
+    test('test1', async () => {
+      console.log('test1-done');
+    });
+    test('test2', async () => {
+      console.log('test2-done');
+    });
+    test('test3', async () => {
+      console.log('test3-done');
+    });
+  `,
+  'b.spec.ts': `
+      const { test } = folio;
+    test('test4', async () => {
+      console.log('test4-done');
+    });
+  `,
+};
+
+test('should respect shard=1/2', async ({ runInlineTest }) => {
+  const result = await runInlineTest(tests, { shard: '1/2' });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+  expect(result.skipped).toBe(1);
+  expect(result.output).toContain('test1-done');
+  expect(result.output).toContain('test2-done');
+  expect(result.output).toContain('test3-done');
+});
+
+test('should respect shard=2/2', async ({ runInlineTest }) => {
+  const result = await runInlineTest(tests, { shard: '2/2' });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.skipped).toBe(3);
+  expect(result.output).toContain('test4-done');
+});

tests/playwright-test/stdio.spec.ts

@@ -0,0 +1,62 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should get top level stdio', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = folio;
+      console.log('\\n%% top level stdout');
+      console.error('\\n%% top level stderr');
+      test('is a test', () => {
+        console.log('\\n%% stdout in a test');
+        console.error('\\n%% stderr in a test');
+      });
+    `
+  });
+  expect(result.output.split('\n').filter(x => x.startsWith('%%'))).toEqual([
+    '%% top level stdout',
+    '%% top level stderr',
+    '%% top level stdout', // top level logs appear twice, because the file is required twice
+    '%% top level stderr',
+    '%% stdout in a test',
+    '%% stderr in a test'
+  ]);
+});
+
+test('should get stdio from env afterAll', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      export const test = folio.test.extend({
+        fixture: [ async ({}, run) => {
+          console.log('\\n%% worker setup');
+          await run();
+          console.log('\\n%% worker teardown');
+        }, { scope: 'worker' } ]
+      });
+    `,
+    'a.spec.js': `
+      const { test } = require('./helper');
+      test('is a test', async ({fixture}) => {});
+    `
+  });
+  expect(result.output.split('\n').filter(x => x.startsWith('%%'))).toEqual([
+    '%% worker setup',
+    '%% worker teardown'
+  ]);
+});
+

tests/playwright-test/test-extend.spec.ts

@@ -0,0 +1,178 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('test.extend should work', async ({ runInlineTest }) => {
+  const { output, passed } = await runInlineTest({
+    'helper.ts': `
+      global.logs = [];
+
+      function createDerivedFixtures(suffix) {
+        return {
+          derivedWorker: [async ({ baseWorker }, run) => {
+            global.logs.push('beforeAll-' + suffix);
+            await run();
+            global.logs.push('afterAll-' + suffix);
+            if (suffix.includes('base'))
+              console.log(global.logs.join('\\n'));
+          }, { scope: 'worker' }],
+
+          derivedTest: async ({ baseTest, derivedWorker }, run) => {
+            global.logs.push('beforeEach-' + suffix);
+            await run();
+            global.logs.push('afterEach-' + suffix);
+          },
+        };
+      }
+
+      export const base = folio.test.declare();
+      export const test1 = base.extend(createDerivedFixtures('e1'));
+      export const test2 = base.extend(createDerivedFixtures('e2'));
+    `,
+    'playwright.config.ts': `
+      import { base } from './helper';
+
+      function createBaseFixtures(suffix) {
+        return {
+          baseWorker: [async ({}, run) => {
+            global.logs.push('beforeAll-' + suffix);
+            await run();
+            global.logs.push('afterAll-' + suffix);
+            if (suffix.includes('base'))
+              console.log(global.logs.join('\\n'));
+          }, { scope: 'worker' }],
+
+          baseTest: async ({ derivedWorker }, run) => {
+            global.logs.push('beforeEach-' + suffix);
+            await run();
+            global.logs.push('afterEach-' + suffix);
+          },
+        };
+      }
+
+      module.exports = { projects: [
+        { define: { test: base, fixtures: createBaseFixtures('base1') } },
+        { define: { test: base, fixtures: createBaseFixtures('base2') } },
+      ] };
+    `,
+    'a.test.ts': `
+      import { test1, test2 } from './helper';
+      test1('should work', async ({ derivedTest }) => {
+        global.logs.push('test1');
+      });
+      test2('should work', async ({ derivedTest }) => {
+        global.logs.push('test2');
+      });
+    `,
+  });
+  expect(passed).toBe(4);
+  expect(output).toContain([
+    'beforeAll-base1',
+    'beforeAll-e1',
+    'beforeEach-base1',
+    'beforeEach-e1',
+    'test1',
+    'afterEach-e1',
+    'afterEach-base1',
+    'afterAll-e1',
+    'afterAll-base1',
+  ].join('\n'));
+  expect(output).toContain([
+    'beforeAll-base1',
+    'beforeAll-e2',
+    'beforeEach-base1',
+    'beforeEach-e2',
+    'test2',
+    'afterEach-e2',
+    'afterEach-base1',
+    'afterAll-e2',
+    'afterAll-base1',
+  ].join('\n'));
+  expect(output).toContain([
+    'beforeAll-base2',
+    'beforeAll-e1',
+    'beforeEach-base2',
+    'beforeEach-e1',
+    'test1',
+    'afterEach-e1',
+    'afterEach-base2',
+    'afterAll-e1',
+    'afterAll-base2',
+  ].join('\n'));
+  expect(output).toContain([
+    'beforeAll-base2',
+    'beforeAll-e2',
+    'beforeEach-base2',
+    'beforeEach-e2',
+    'test2',
+    'afterEach-e2',
+    'afterEach-base2',
+    'afterAll-e2',
+    'afterAll-base2',
+  ].join('\n'));
+});
+
+test('test.declare should be inserted at the right place', async ({ runInlineTest }) => {
+  const { output, passed } = await runInlineTest({
+    'helper.ts': `
+      const test1 = folio.test.extend({
+        foo: async ({}, run) => {
+          console.log('before-foo');
+          await run('foo');
+          console.log('after-foo');
+        },
+      });
+      export const test2 = test1.declare<{ bar: string }>();
+      export const test3 = test2.extend({
+        baz: async ({ bar }, run) => {
+          console.log('before-baz');
+          await run(bar + 'baz');
+          console.log('after-baz');
+        },
+      });
+    `,
+    'playwright.config.ts': `
+      import { test2 } from './helper';
+      const fixtures = {
+        bar: async ({ foo }, run) => {
+          console.log('before-bar');
+          await run(foo + 'bar');
+          console.log('after-bar');
+        },
+      };
+      module.exports = {
+        define: { test: test2, fixtures },
+      };
+    `,
+    'a.test.js': `
+      const { test3 } = require('./helper');
+      test3('should work', async ({baz}) => {
+        console.log('test-' + baz);
+      });
+    `,
+  });
+  expect(passed).toBe(1);
+  expect(output).toContain([
+    'before-foo',
+    'before-bar',
+    'before-baz',
+    'test-foobarbaz',
+    'after-baz',
+    'after-bar',
+    'after-foo',
+  ].join('\n'));
+});

tests/playwright-test/test-ignore.spec.ts

@@ -0,0 +1,262 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+import * as path from 'path';
+
+const tests = {
+  'a.test.ts': `
+    const { test } = folio;
+    test('pass', ({}) => {});
+  `,
+  'b.test.ts': `
+    const { test } = folio;
+    test('pass', ({}) => {});
+  `,
+  'c.test.ts': `
+    const { test } = folio;
+    test('pass', ({}) => {});
+  `
+};
+
+test('should run all three tests', async ({ runInlineTest }) => {
+  const result = await runInlineTest(tests);
+  expect(result.passed).toBe(3);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should ignore a test', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    ...tests,
+    'playwright.config.ts': `
+      module.exports = { testIgnore: 'b.test.ts' };
+    `,
+  });
+  expect(result.passed).toBe(2);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should ignore a folder', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { testIgnore: 'folder/**' };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'folder/a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'folder/b.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'folder/c.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  });
+  expect(result.passed).toBe(1);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should ignore a node_modules', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'node_modules/a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'node_modules/b.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'folder/c.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  });
+  expect(result.passed).toBe(2);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should filter tests', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    ...tests,
+    'playwright.config.ts': `
+      module.exports = { testIgnore: 'c.test.*' };
+    `,
+  });
+  expect(result.passed).toBe(2);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should use a different test match', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    ...tests,
+    'playwright.config.ts': `
+      module.exports = { testMatch: '[a|b].test.ts' };
+    `,
+  });
+  expect(result.passed).toBe(2);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should use an array for testMatch', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { testMatch: ['b.test.ts', /\\${path.sep}a.[tes]{4}.TS$/i] };
+    `,
+    'dir/a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'b.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'c.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  });
+  expect(result.passed).toBe(2);
+  expect(result.report.suites.map(s => s.file).sort()).toEqual(['b.test.ts', 'dir/a.test.ts']);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should match absolute path', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = { testDir: path.join(__dirname, 'dir'), testMatch: /dir\\${path.sep}a/ };
+    `,
+    'dir/a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'dir/b.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  });
+  expect(result.passed).toBe(1);
+  expect(result.report.suites.map(s => s.file).sort()).toEqual(['a.test.ts']);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should match cli string argument', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      import * as path from 'path';
+      module.exports = { testDir: path.join(__dirname, 'dir') };
+    `,
+    'dir/a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'dir/b.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  }, { args: [`dir\\${path.sep}a`] });
+  expect(result.passed).toBe(1);
+  expect(result.report.suites.map(s => s.file).sort()).toEqual(['a.test.ts']);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should match regex string argument', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'dir/filea.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'dir/fileb.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'filea.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  }, { args: ['/filea.*ts/'] });
+  expect(result.passed).toBe(2);
+  expect(result.report.suites.map(s => s.file).sort()).toEqual(['dir/filea.test.ts', 'filea.test.ts']);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should match by directory', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'dir-a/file.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'dir-b/file1.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'dir-b/file2.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'file.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  }, { args: ['dir-b'] });
+  expect(result.passed).toBe(2);
+  expect(result.report.suites.map(s => s.file).sort()).toEqual(['dir-b/file1.test.ts', 'dir-b/file2.test.ts']);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should ignore node_modules even with custom testIgnore', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { testIgnore: 'a.test.ts' };
+    `,
+    'a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'node_modules/a.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'node_modules/b.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `,
+    'folder/c.test.ts': `
+      const { test } = folio;
+      test('pass', ({}) => {});
+    `
+  });
+  expect(result.passed).toBe(1);
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/test-info.spec.ts

@@ -0,0 +1,54 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should work directly', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('test 1', async ({}, testInfo) => {
+        expect(testInfo.title).toBe('test 1');
+      });
+      test('test 2', async ({}, testInfo) => {
+        expect(testInfo.title).toBe('test 2');
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should work via fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      export const test = folio.test.extend({
+        title: async ({}, run, testInfo) => {
+          await run(testInfo.title);
+        },
+      });
+    `,
+    'a.test.js': `
+      const { test } = require('./helper');
+      test('test 1', async ({title}) => {
+        expect(title).toBe('test 1');
+      });
+      test('test 2', async ({title}) => {
+        expect(title).toBe('test 2');
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/test-modifiers.spec.ts

@@ -0,0 +1,216 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('test modifiers should work', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      export const test = folio.test.extend({
+        foo: true,
+      });
+    `,
+    'a.test.ts': `
+      import { test } from './helper';
+
+      test('passed1', async ({foo}) => {
+      });
+      test('passed2', async ({foo}) => {
+        test.skip(false);
+      });
+      test('passed3', async () => {
+        test.fixme(undefined);
+      });
+      test('passed4', async () => {
+        test.fixme(undefined, 'reason')
+      });
+      test('passed5', async ({foo}) => {
+        test.skip(false);
+      });
+
+      test('skipped1', async ({foo}) => {
+        test.skip();
+      });
+      test('skipped2', async ({foo}) => {
+        test.skip('reason');
+      });
+      test('skipped3', async ({foo}) => {
+        test.skip(foo);
+      });
+      test('skipped4', async ({foo}) => {
+        test.skip(foo, 'reason');
+      });
+      test('skipped5', async () => {
+        test.fixme();
+      });
+      test('skipped6', async () => {
+        test.fixme(true, 'reason');
+      });
+
+      test('failed1', async ({foo}) => {
+        test.fail();
+        expect(true).toBe(false);
+      });
+      test('failed2', async ({foo}) => {
+        test.fail('reason');
+        expect(true).toBe(false);
+      });
+      test('failed3', async ({foo}) => {
+        test.fail(foo);
+        expect(true).toBe(false);
+      });
+      test('failed4', async ({foo}) => {
+        test.fail(foo, 'reason');
+        expect(true).toBe(false);
+      });
+
+      test.describe('suite1', () => {
+        test.skip();
+        test('suite1', () => {});
+      });
+
+      test.describe('suite2', () => {
+        test.skip(true);
+        test('suite2', () => {});
+      });
+
+      test.describe('suite3', () => {
+        test.skip(({ foo }) => foo, 'reason');
+        test('suite3', () => {});
+      });
+
+      test.describe('suite3', () => {
+        test.skip(({ foo }) => !foo, 'reason');
+        test('suite4', () => {});
+      });
+    `,
+  });
+
+  const expectTest = (title: string, expectedStatus: string, status: string, annotations: any) => {
+    const spec = result.report.suites[0].specs.find(s => s.title === title) ||
+        result.report.suites[0].suites.find(s => s.specs[0].title === title).specs[0];
+    const test = spec.tests[0];
+    expect(test.expectedStatus).toBe(expectedStatus);
+    expect(test.results[0].status).toBe(status);
+    expect(test.annotations).toEqual(annotations);
+  };
+  expectTest('passed1', 'passed', 'passed', []);
+  expectTest('passed2', 'passed', 'passed', []);
+  expectTest('passed3', 'passed', 'passed', []);
+  expectTest('passed4', 'passed', 'passed', []);
+  expectTest('passed5', 'passed', 'passed', []);
+  expectTest('skipped1', 'skipped', 'skipped', [{ type: 'skip' }]);
+  expectTest('skipped2', 'skipped', 'skipped', [{ type: 'skip' }]);
+  expectTest('skipped3', 'skipped', 'skipped', [{ type: 'skip' }]);
+  expectTest('skipped4', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]);
+  expectTest('skipped5', 'skipped', 'skipped', [{ type: 'fixme' }]);
+  expectTest('skipped6', 'skipped', 'skipped', [{ type: 'fixme', description: 'reason' }]);
+  expectTest('failed1', 'failed', 'failed', [{ type: 'fail' }]);
+  expectTest('failed2', 'failed', 'failed', [{ type: 'fail' }]);
+  expectTest('failed3', 'failed', 'failed', [{ type: 'fail' }]);
+  expectTest('failed4', 'failed', 'failed', [{ type: 'fail', description: 'reason' }]);
+  expectTest('suite1', 'skipped', 'skipped', [{ type: 'skip' }]);
+  expectTest('suite2', 'skipped', 'skipped', [{ type: 'skip' }]);
+  expectTest('suite3', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]);
+  expectTest('suite4', 'passed', 'passed', []);
+  expect(result.passed).toBe(10);
+  expect(result.skipped).toBe(9);
+});
+
+test('test modifiers should check types', async ({runTSC}) => {
+  const result = await runTSC({
+    'helper.ts': `
+      export const test = folio.test.extend<{ foo: boolean }>({
+        foo: async ({}, use, testInfo) => {
+          testInfo.skip();
+          testInfo.fixme(false);
+          testInfo.slow(true, 'reason');
+          testInfo.fail(false, 'reason');
+          // @ts-expect-error
+          testInfo.skip('reason');
+          // @ts-expect-error
+          testInfo.fixme('foo', 'reason');
+          // @ts-expect-error
+          testInfo.slow(() => true);
+          use(true);
+        },
+      });
+    `,
+    'a.test.ts': `
+      import { test } from './helper';
+
+      test('passed1', async ({foo}) => {
+        test.skip();
+      });
+      test('passed2', async ({foo}) => {
+        test.skip(foo);
+      });
+      test('passed2', async ({foo}) => {
+        test.skip(foo, 'reason');
+      });
+      test('passed3', async ({foo}) => {
+        test.skip(({foo}) => foo);
+      });
+      test('passed3', async ({foo}) => {
+        test.skip(({foo}) => foo, 'reason');
+      });
+      test('passed3', async ({foo}) => {
+        // @ts-expect-error
+        test.skip('foo', 'bar');
+      });
+      test('passed3', async ({foo}) => {
+        // @ts-expect-error
+        test.skip(({ bar }) => bar, 'reason');
+      });
+      test('passed3', async ({foo}) => {
+        // @ts-expect-error
+        test.skip(42);
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should skip inside fixture', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const test = folio.test.extend({
+        foo: async ({}, run, testInfo) => {
+          testInfo.skip(true, 'reason');
+          await run();
+        },
+      });
+
+      test('skipped', async ({ foo }) => {
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.skipped).toBe(1);
+  expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+});
+
+test('modifier with a function should throw in the test', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      folio.test('skipped', async ({}) => {
+        folio.test.skip(() => true);
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('test.skip() with a function can only be called inside describe block');
+});

tests/playwright-test/test-output-dir.spec.ts

@@ -0,0 +1,233 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { test, expect } from './playwright-test-fixtures';
+
+test('should work and remove non-failures on CI', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'dir/my-test.spec.js': `
+      const { test } = folio;
+      test('test 1', async ({}, testInfo) => {
+        if (testInfo.retry) {
+          expect(testInfo.outputDir).toContain('dir-my-test-test-1-retry' + testInfo.retry);
+          expect(testInfo.outputPath('foo', 'bar')).toContain(require('path').join('dir-my-test-test-1-retry' + testInfo.retry, 'foo', 'bar'));
+          require('fs').writeFileSync(testInfo.outputPath('file.txt'), 'content', 'utf-8');
+        } else {
+          expect(testInfo.outputDir).toContain('dir-my-test-test-1');
+          expect(testInfo.outputPath()).toContain('dir-my-test-test-1');
+          expect(testInfo.outputPath('foo', 'bar')).toContain(require('path').join('dir-my-test-test-1', 'foo', 'bar'));
+          require('fs').writeFileSync(testInfo.outputPath('file.txt'), 'content', 'utf-8');
+        }
+        expect(require('fs').existsSync(testInfo.outputDir)).toBe(true);
+        if (testInfo.retry < 2)
+          throw new Error('Give me retries');
+      });
+    `,
+  }, { retries: 2 }, { CI: '1' });
+  expect(result.exitCode).toBe(0);
+
+  expect(result.results[0].status).toBe('failed');
+  expect(result.results[0].retry).toBe(0);
+  // Should only fail the last retry check.
+  expect(result.results[0].error.message).toBe('Give me retries');
+
+  expect(result.results[1].status).toBe('failed');
+  expect(result.results[1].retry).toBe(1);
+  // Should only fail the last retry check.
+  expect(result.results[1].error.message).toBe('Give me retries');
+
+  expect(result.results[2].status).toBe('passed');
+  expect(result.results[2].retry).toBe(2);
+
+  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1'))).toBe(true);
+  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1-retry1'))).toBe(true);
+  // Last retry is successfull, so output dir should be removed.
+  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1-retry2'))).toBe(false);
+});
+
+test('should include repeat token', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'a.spec.js': `
+      const { test } = folio;
+      test('test', ({}, testInfo) => {
+        if (testInfo.repeatEachIndex)
+          expect(testInfo.outputPath('')).toContain('repeat' + testInfo.repeatEachIndex);
+        else
+          expect(testInfo.outputPath('')).not.toContain('repeat' + testInfo.repeatEachIndex);
+      });
+    `
+  }, { 'repeat-each': 3 });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+});
+
+test('should include the project name', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      export const test = folio.test.extend({
+        auto: [ async ({}, run, testInfo) => {
+          await run();
+        }, { auto: true } ]
+      });
+      export const test2 = folio.test.extend({
+        auto: [ async ({}, run, testInfo) => {
+          testInfo.snapshotSuffix = 'suffix';
+          await run();
+        }, { auto: true } ]
+      });
+    `,
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        {},
+        { name: 'foo' },
+        { name: 'foo' },
+        { name: 'Bar space!' },
+      ] };
+    `,
+    'my-test.spec.js': `
+      const { test, test2 } = require('./helper');
+      test('test 1', async ({}, testInfo) => {
+        console.log(testInfo.outputPath('bar.txt').replace(/\\\\/g, '/'));
+        console.log(testInfo.snapshotPath('bar.txt').replace(/\\\\/g, '/'));
+        if (testInfo.retry !== 1)
+          throw new Error('Give me a retry');
+      });
+      test2('test 2', async ({}, testInfo) => {
+        console.log(testInfo.outputPath('bar.txt').replace(/\\\\/g, '/'));
+        console.log(testInfo.snapshotPath('bar.txt').replace(/\\\\/g, '/'));
+      });
+    `,
+  }, { retries: 1 });
+  expect(result.exitCode).toBe(0);
+  expect(result.results[0].status).toBe('failed');
+  expect(result.results[1].status).toBe('passed');
+
+  // test1, run with empty
+  expect(result.output).toContain('test-results/my-test-test-1/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar.txt');
+  expect(result.output).toContain('test-results/my-test-test-1-retry1/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar.txt');
+
+  // test1, run with foo #1
+  expect(result.output).toContain('test-results/my-test-test-1-foo1/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo.txt');
+  expect(result.output).toContain('test-results/my-test-test-1-foo1-retry1/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo.txt');
+
+  // test1, run with foo #2
+  expect(result.output).toContain('test-results/my-test-test-1-foo2/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo.txt');
+  expect(result.output).toContain('test-results/my-test-test-1-foo2-retry1/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo.txt');
+
+  // test1, run with bar
+  expect(result.output).toContain('test-results/my-test-test-1-Bar-space-/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-Bar-space-.txt');
+  expect(result.output).toContain('test-results/my-test-test-1-Bar-space--retry1/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-Bar-space-.txt');
+
+  // test2, run with empty
+  expect(result.output).toContain('test-results/my-test-test-2/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-suffix.txt');
+
+  // test2, run with foo #1
+  expect(result.output).toContain('test-results/my-test-test-2-foo1/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo-suffix.txt');
+
+  // test2, run with foo #2
+  expect(result.output).toContain('test-results/my-test-test-2-foo2/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-foo-suffix.txt');
+
+  // test2, run with bar
+  expect(result.output).toContain('test-results/my-test-test-2-Bar-space-/bar.txt');
+  expect(result.output).toContain('my-test.spec.js-snapshots/bar-Bar-space--suffix.txt');
+});
+
+test('should remove output dirs for projects run', async ({runInlineTest}, testInfo) => {
+  const paths: string[] = [];
+  const files: string[] = [];
+
+  for (let i = 0; i < 3; i++) {
+    const p = testInfo.outputPath('path' + i);
+    await fs.promises.mkdir(p, { recursive: true });
+    const f = path.join(p, 'my-file.txt');
+    await fs.promises.writeFile(f, 'contents', 'utf-8');
+    paths.push(p);
+    files.push(f);
+  }
+
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = { projects: [
+        { outputDir: ${JSON.stringify(paths[0])} },
+        { outputDir: ${JSON.stringify(paths[2])} },
+      ] };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('my test', ({}, testInfo) => {});
+    `
+  }, { output: '' });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+
+  expect(fs.existsSync(files[0])).toBe(false);
+  expect(fs.existsSync(files[1])).toBe(true);
+  expect(fs.existsSync(files[2])).toBe(false);
+});
+
+test('should remove folders with preserveOutput=never', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      export default { preserveOutput: 'never' };
+    `,
+    'dir/my-test.spec.js': `
+      const { test } = folio;
+      test('test 1', async ({}, testInfo) => {
+        require('fs').writeFileSync(testInfo.outputPath('file.txt'), 'content', 'utf-8');
+        if (testInfo.retry < 2)
+          throw new Error('Give me retries');
+      });
+    `,
+  }, { retries: 2 });
+  expect(result.exitCode).toBe(0);
+  expect(result.results.length).toBe(3);
+
+  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1'))).toBe(false);
+  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1-retry1'))).toBe(false);
+  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1-retry2'))).toBe(false);
+});
+
+test('should not remove folders on non-CI', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'dir/my-test.spec.js': `
+      const { test } = folio;
+      test('test 1', async ({}, testInfo) => {
+        require('fs').writeFileSync(testInfo.outputPath('file.txt'), 'content', 'utf-8');
+        if (testInfo.retry < 2)
+          throw new Error('Give me retries');
+      });
+    `,
+  }, { 'retries': 2 }, { CI: '' });
+  expect(result.exitCode).toBe(0);
+  expect(result.results.length).toBe(3);
+
+  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1'))).toBe(true);
+  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1-retry1'))).toBe(true);
+  expect(fs.existsSync(testInfo.outputPath('test-results', 'dir-my-test-test-1-retry2'))).toBe(true);
+});

tests/playwright-test/timeout.spec.ts

@@ -0,0 +1,113 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should run fixture teardown on timeout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'helper.ts': `
+      export const test = folio.test.extend({
+        foo: async ({}, run, testInfo) => {
+          await run();
+          console.log('STATUS:' + testInfo.status);
+        }
+      });
+    `,
+    'c.spec.ts': `
+      import { test } from './helper';
+      test('works', async ({ foo }) => {
+        await new Promise(f => setTimeout(f, 100000));
+      });
+    `
+  }, { timeout: 1000 });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('STATUS:timedOut');
+});
+
+test('should respect test.setTimeout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = folio;
+      test('fails', async ({}) => {
+        await new Promise(f => setTimeout(f, 1500));
+      });
+      test('passes', async ({}) => {
+        await new Promise(f => setTimeout(f, 500));
+        test.setTimeout(2000);
+        await new Promise(f => setTimeout(f, 1000));
+      });
+
+      test.describe('suite', () => {
+        test.beforeEach(() => {
+          test.setTimeout(2000);
+        });
+        test('passes2', async ({}, testInfo) => {
+          expect(testInfo.timeout).toBe(2000);
+          await new Promise(f => setTimeout(f, 1500));
+        });
+      });
+    `
+  }, { timeout: 1000 });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.passed).toBe(2);
+  expect(result.output).toContain('Timeout of 1000ms exceeded');
+});
+
+test('should timeout when calling test.setTimeout too late', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = folio;
+      test('fails', async ({}) => {
+        await new Promise(f => setTimeout(f, 500));
+        test.setTimeout(100);
+        await new Promise(f => setTimeout(f, 1));
+      });
+    `
+  }, { timeout: 1000 });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.output).toContain('Timeout of 100ms exceeded');
+});
+
+test('should respect test.slow', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = folio;
+      test('fails', async ({}) => {
+        await new Promise(f => setTimeout(f, 1500));
+      });
+      test('passes', async ({}) => {
+        test.slow();
+        await new Promise(f => setTimeout(f, 1500));
+      });
+
+      test.describe('suite', () => {
+        test.slow();
+        test('passes2', async ({}, testInfo) => {
+          expect(testInfo.timeout).toBe(3000);
+          await new Promise(f => setTimeout(f, 1500));
+        });
+      });
+    `
+  }, { timeout: 1000 });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.passed).toBe(2);
+  expect(result.output).toContain('Timeout of 1000ms exceeded');
+});

tests/playwright-test/types-2.spec.ts

@@ -0,0 +1,107 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('basics should work', async ({runTSC}) => {
+  const result = await runTSC({
+    'a.spec.ts': `
+      const { test } = folio;
+      test.describe('suite', () => {
+        test.beforeEach(async () => {});
+        test('my test', async({}, testInfo) => {
+          expect(testInfo.title).toBe('my test');
+          testInfo.annotations[0].type;
+        });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('can pass sync functions everywhere', async ({runTSC}) => {
+  const result = await runTSC({
+    'a.spec.ts': `
+      const test = folio.test.extend<{ foo: string }>({
+        foo: ({}, use) => use('bar'),
+      });
+      test.beforeEach(({ foo }) => {});
+      test.afterEach(({ foo }) => {});
+      test.beforeAll(() => {});
+      test.afterAll(() => {});
+      test('my test', ({ foo }) => {});
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('can return anything from hooks', async ({runTSC}) => {
+  const result = await runTSC({
+    'a.spec.ts': `
+      const { test } = folio;
+      test.beforeEach(() => '123');
+      test.afterEach(() => 123);
+      test.beforeAll(() => [123]);
+      test.afterAll(() => ({ a: 123 }));
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('test.declare should check types', async ({runTSC}) => {
+  const result = await runTSC({
+    'helper.ts': `
+      export const test = folio.test;
+      export const test1 = test.declare<{ foo: string }>();
+      export const test2 = test1.extend<{ bar: number }>({
+        bar: async ({ foo }, run) => { await run(parseInt(foo)); }
+      });
+      export const test3 = test1.extend<{ bar: number }>({
+        // @ts-expect-error
+        bar: async ({ baz }, run) => { await run(42); }
+      });
+    `,
+    'playwright.config.ts': `
+      import { test1 } from './helper';
+      const configs: folio.Config[] = [];
+      configs.push({});
+      configs.push({
+        define: {
+          test: test1,
+          fixtures: { foo: 'foo' }
+        },
+      });
+
+      configs.push({
+        // @ts-expect-error
+        define: { test: {}, fixtures: {} },
+      });
+      module.exports = configs;
+    `,
+    'a.spec.ts': `
+      import { test, test1, test2, test3 } from './helper';
+      // @ts-expect-error
+      test('my test', async ({ foo }) => {});
+      test1('my test', async ({ foo }) => {});
+      // @ts-expect-error
+      test1('my test', async ({ foo, bar }) => {});
+      test2('my test', async ({ foo, bar }) => {});
+      // @ts-expect-error
+      test2('my test', async ({ foo, baz }) => {});
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/types.spec.ts

@@ -0,0 +1,165 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('sanity', async ({runTSC}) => {
+  const result = await runTSC({
+    'a.spec.ts': `
+      const { test } = folio;
+      // @ts-expect-error
+      test.foo();
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should check types of fixtures', async ({runTSC}) => {
+  const result = await runTSC({
+    'helper.ts': `
+      export type MyOptions = { foo: string, bar: number };
+      export const test = folio.test.extend<{ foo: string }, { bar: number }>({
+        foo: 'foo',
+        bar: [ 42, { scope: 'worker' } ],
+      });
+
+      const good1 = test.extend<{}>({ foo: async ({ bar }, run) => run('foo') });
+      const good2 = test.extend<{}>({ bar: ({}, run) => run(42) });
+      const good3 = test.extend<{}>({ bar: ({}, run) => run(42) });
+      const good4 = test.extend<{}>({ bar: async ({ bar }, run) => run(42) });
+      const good5 = test.extend<{}>({ foo: async ({ foo }, run) => run('foo') });
+      const good6 = test.extend<{ baz: boolean }>({
+        baz: false,
+        foo: async ({ baz }, run) => run('foo')
+      });
+      const good7 = test.extend<{ baz: boolean }>({
+        baz: [ false, { auto: true } ],
+      });
+
+      // @ts-expect-error
+      const fail1 = test.extend<{}>({ foo: 42 });
+      // @ts-expect-error
+      const fail2 = test.extend<{}>({ bar: async ({ foo }, run) => run(42) });
+      // @ts-expect-error
+      const fail3 = test.extend<{}>({ baz: 42 });
+      // @ts-expect-error
+      const fail4 = test.extend<{}>({ foo: async ({ foo }, run) => run(42) });
+      // @ts-expect-error
+      const fail5 = test.extend<{}>({ bar: async ({}, run) => run('foo') });
+      const fail6 = test.extend<{ baz: boolean }>({
+        // @ts-expect-error
+        baz: [ true, { scope: 'worker' } ],
+      });
+      const fail7 = test.extend<{}, { baz: boolean }>({
+        // @ts-expect-error
+        baz: [ true, { scope: 'test' } ],
+      });
+      const fail8 = test.extend<{}, { baz: boolean }>({
+        // @ts-expect-error
+        baz: true,
+      });
+    `,
+    'playwright.config.ts': `
+      import { MyOptions } from './helper';
+      const configs1: folio.Config[] = [];
+      configs1.push({ use: { foo: '42', bar: 42 } });
+      configs1.push({ use: { foo: '42', bar: 42 }, timeout: 100 });
+
+      const configs2: folio.Config<MyOptions>[] = [];
+      configs2.push({ use: { foo: '42', bar: 42 } });
+      // @ts-expect-error
+      folio.runTests({ use: { foo: '42', bar: 42 } }, {});
+      // @ts-expect-error
+      configs2.push({ use: { bar: '42' } });
+      // @ts-expect-error
+      configs2.push(new Env2());
+      // @ts-expect-error
+      configs2.push({ use: { foo: 42, bar: 42 } });
+      // @ts-expect-error
+      configs2.push({ beforeAll: async () => { return {}; } });
+      // TODO: next line should not compile.
+      configs2.push({ timeout: 100 });
+      // @ts-expect-error
+      configs2.push('alias');
+      // TODO: next line should not compile.
+      configs2.push({});
+    `,
+    'a.spec.ts': `
+      import { test } from './helper';
+      test.use({ foo: 'foo' });
+      test.use({});
+
+      // @ts-expect-error
+      test.use({ foo: 42 });
+      // @ts-expect-error
+      test.use({ baz: 'baz' });
+
+      test('my test', async ({ foo, bar }) => {
+        bar += parseInt(foo);
+      });
+      test('my test', ({ foo, bar }) => {
+        bar += parseInt(foo);
+      });
+      test('my test', () => {});
+      // @ts-expect-error
+      test('my test', async ({ a }) => {
+      });
+
+      // @ts-expect-error
+      test.beforeEach(async ({ a }) => {});
+      test.beforeEach(async ({ foo, bar }) => {});
+      test.beforeEach(() => {});
+
+      // @ts-expect-error
+      test.beforeAll(async ({ a }) => {});
+      // @ts-expect-error
+      test.beforeAll(async ({ foo, bar }) => {});
+      test.beforeAll(async ({ bar }) => {});
+      test.beforeAll(() => {});
+
+      // @ts-expect-error
+      test.afterEach(async ({ a }) => {});
+      test.afterEach(async ({ foo, bar }) => {});
+      test.afterEach(() => {});
+
+      // @ts-expect-error
+      test.afterAll(async ({ a }) => {});
+      // @ts-expect-error
+      test.afterAll(async ({ foo, bar }) => {});
+      test.afterAll(async ({ bar }) => {});
+      test.afterAll(() => {});
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('config should allow void/empty options', async ({runTSC}) => {
+  const result = await runTSC({
+    'playwright.config.ts': `
+      const configs: folio.Config[] = [];
+      configs.push({});
+      configs.push({ timeout: 100 });
+      configs.push();
+      configs.push({ use: { foo: 42 }});
+    `,
+    'a.spec.ts': `
+      const { test } = folio;
+      test('my test', async () => {
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/worker-index.spec.ts

@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should run in parallel', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    '1.spec.ts': `
+      import * as fs from 'fs';
+      import * as path from 'path';
+      const { test } = folio;
+      test('succeeds', async ({}, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+        // First test waits for the second to start to work around the race.
+        while (true) {
+          if (fs.existsSync(path.join(testInfo.project.outputDir, 'parallel-index.txt')))
+            break;
+          await new Promise(f => setTimeout(f, 100));
+        }
+      });
+    `,
+    '2.spec.ts': `
+      import * as fs from 'fs';
+      import * as path from 'path';
+      const { test } = folio;
+      test('succeeds', async ({}, testInfo) => {
+        // First test waits for the second to start to work around the race.
+        fs.mkdirSync(testInfo.project.outputDir, { recursive: true });
+        fs.writeFileSync(path.join(testInfo.project.outputDir, 'parallel-index.txt'), 'TRUE');
+        expect(testInfo.workerIndex).toBe(1);
+      });
+    `,
+  });
+  expect(result.passed).toBe(2);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should reuse worker for multiple tests', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = folio;
+      test('succeeds', async ({}, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+      });
+
+      test('succeeds', async ({}, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+      });
+
+      test('succeeds', async ({}, testInfo) => {
+        expect(testInfo.workerIndex).toBe(0);
+      });
+    `,
+  });
+  expect(result.passed).toBe(3);
+  expect(result.exitCode).toBe(0);
+});
+
+test('should not reuse worker for different suites', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [{}, {}, {}] };
+    `,
+    'a.test.js': `
+      const { test } = folio;
+      test('succeeds', async ({}, testInfo) => {
+        console.log('workerIndex-' + testInfo.workerIndex);
+      });
+    `,
+  });
+  expect(result.passed).toBe(3);
+  expect(result.exitCode).toBe(0);
+  expect(result.results.map(r => r.workerIndex).sort()).toEqual([0, 1, 2]);
+  expect(result.output).toContain('workerIndex-0');
+  expect(result.output).toContain('workerIndex-1');
+  expect(result.output).toContain('workerIndex-2');
+});

types/test.d.ts

@@ -15,7 +15,8 @@
  */
 
 import type { Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials } from './types';
-import type { Project, Config } from 'folio';
+import type { Project, Config, TestType } from './testInternal';
+import type { Expect } from './testExpect';
 
 /**
  * The name of the browser supported by Playwright.
@@ -279,9 +280,7 @@ export type PlaywrightTestArgs = {
 export type PlaywrightTestProject<TestArgs = {}, WorkerArgs = {}> = Project<PlaywrightTestOptions & TestArgs, PlaywrightWorkerOptions & WorkerArgs>;
 export type PlaywrightTestConfig<TestArgs = {}, WorkerArgs = {}> = Config<PlaywrightTestOptions & TestArgs, PlaywrightWorkerOptions & WorkerArgs>;
 
-export * from 'folio';
+export type { Project, Config, TestStatus, TestInfo, WorkerInfo, TestType, Fixtures, TestFixture, WorkerFixture } from './testInternal';
-
-import type { TestType } from 'folio';
 
 /**
  * These tests are executed in Playwright environment that launches the browser
@@ -289,3 +288,5 @@ import type { TestType } from 'folio';
  */
 export const test: TestType<PlaywrightTestArgs & PlaywrightTestOptions, PlaywrightWorkerArgs & PlaywrightWorkerOptions>;
 export default test;
+
+export const expect: Expect;

types/testExpect.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ * Modifications copyright (c) Microsoft Corporation.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type expect from 'expect';
+import type { ExpectedAssertionsErrors } from 'expect/build/types';
+
+export declare type AsymmetricMatcher = Record<string, any>;
+
+export declare type Expect = {
+  <T = unknown>(actual: T): folio.Matchers<T>;
+
+  // Sourced from node_modules/expect/build/types.d.ts
+  assertions(arg0: number): void;
+  extend(arg0: any): void;
+  extractExpectedAssertionsErrors: () => ExpectedAssertionsErrors;
+  getState(): expect.MatcherState;
+  hasAssertions(): void;
+  setState(state: Partial<expect.MatcherState>): void;
+  any(expectedObject: any): AsymmetricMatcher;
+  anything(): AsymmetricMatcher;
+  arrayContaining(sample: Array<unknown>): AsymmetricMatcher;
+  objectContaining(sample: Record<string, unknown>): AsymmetricMatcher;
+  stringContaining(expected: string): AsymmetricMatcher;
+  stringMatching(expected: string | RegExp): AsymmetricMatcher;
+};
+
+declare global {
+  export namespace jest {
+    export interface Matchers<R> extends expect.Matchers<R> {
+
+    }
+  }
+  export namespace folio {
+    export interface Matchers<R> extends jest.Matchers<R> {
+      /**
+       * If you know how to test something, `.not` lets you test its opposite.
+       */
+      not: folio.Matchers<R>;
+      /**
+       * Use resolves to unwrap the value of a fulfilled promise so any other
+       * matcher can be chained. If the promise is rejected the assertion fails.
+       */
+      resolves: folio.Matchers<Promise<R>>;
+      /**
+      * Unwraps the reason of a rejected promise so any other matcher can be chained.
+      * If the promise is fulfilled the assertion fails.
+      */
+      rejects: folio.Matchers<Promise<R>>;
+      /**
+       * Match snapshot
+       */
+      toMatchSnapshot(options?: {
+        name?: string,
+        threshold?: number
+      }): R;
+      /**
+       * Match snapshot
+       */
+      toMatchSnapshot(name: string, options?: {
+        threshold?: number
+      }): R;
+    }
+  }
+}
+
+export { };

types/testInternal.d.ts

@@ -0,0 +1,868 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Expect } from './testExpect';
+export type { Expect } from './testExpect';
+
+export type ReporterDescription =
+  ['dot'] |
+  ['line'] |
+  ['list'] |
+  ['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] |
+  ['json'] | ['json', { outputFile?: string }] |
+  ['null'] |
+  [string] | [string, any];
+
+export type Shard = { total: number, current: number } | null;
+export type PreserveOutput = 'always' | 'never' | 'failures-only';
+export type UpdateSnapshots = 'all' | 'none' | 'missing';
+
+type FixtureDefine<TestArgs extends KeyValue = {}, WorkerArgs extends KeyValue = {}> = { test: TestType<TestArgs, WorkerArgs>, fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs> };
+
+/**
+ * Test run configuration.
+ */
+interface ProjectBase {
+  /**
+   * Any JSON-serializable metadata that will be put directly to the test report.
+   */
+  metadata?: any;
+
+  /**
+   * The project name, shown in the title of each test.
+   */
+  name?: string;
+
+  /**
+   * Output directory for files created during the test run.
+   */
+  outputDir?: string;
+
+  /**
+   * The number of times to repeat each test, useful for debugging flaky tests.
+   */
+  repeatEach?: number;
+
+  /**
+   * The maximum number of retry attempts given to failed tests.
+   */
+  retries?: number;
+
+  /**
+   * Directory that will be recursively scanned for test files.
+   */
+  testDir?: string;
+
+  /**
+   * Files matching one of these patterns are not executed as test files.
+   * Matching is performed against the absolute file path.
+   * Strings are treated as glob patterns.
+   */
+  testIgnore?: string | RegExp | (string | RegExp)[];
+
+  /**
+   * Only the files matching one of these patterns are executed as test files.
+   * Matching is performed against the absolute file path.
+   * Strings are treated as glob patterns.
+   */
+  testMatch?: string | RegExp | (string | RegExp)[];
+
+  /**
+   * Timeout for each test in milliseconds.
+   */
+  timeout?: number;
+}
+
+/**
+ * Test run configuration.
+ */
+export interface Project<TestArgs = {}, WorkerArgs = {}> extends ProjectBase {
+  /**
+   * Fixtures defined for abstract tests created with `test.declare()` method.
+   */
+  define?: FixtureDefine | FixtureDefine[];
+
+  /**
+   * Fixture overrides for this run. Useful for specifying options.
+   */
+  use?: Fixtures<{}, {}, TestArgs, WorkerArgs>;
+}
+export type FullProject<TestArgs = {}, WorkerArgs = {}> = Required<Project<TestArgs, WorkerArgs>>;
+
+/**
+ * Testing configuration.
+ */
+interface ConfigBase {
+  /**
+   * Whether to exit with an error if any tests are marked as `test.only`. Useful on CI.
+   */
+  forbidOnly?: boolean;
+
+  /**
+   * Path to the global setup file. This file will be required and run before all the tests.
+   * It must export a single function.
+   */
+  globalSetup?: string;
+
+  /**
+   * Path to the global teardown file. This file will be required and run after all the tests.
+   * It must export a single function.
+   */
+  globalTeardown?: string;
+
+  /**
+   * Maximum time in milliseconds the whole test suite can run.
+   */
+  globalTimeout?: number;
+
+  /**
+   * Filter to only run tests with a title matching one of the patterns.
+   */
+  grep?: RegExp | RegExp[];
+
+  /**
+   * The maximum number of test failures for this test run. After reaching this number,
+   * testing will stop and exit with an error. Setting to zero (default) disables this behavior.
+   */
+  maxFailures?: number;
+
+  /**
+   * Whether to preserve test output in the `outputDir`:
+   * - `'always'` - preserve output for all tests;
+   * - `'never'` - do not preserve output for any tests;
+   * - `'failures-only'` - only preserve output for failed tests.
+   */
+  preserveOutput?: PreserveOutput;
+
+  /**
+   * Reporter to use. Available options:
+   * - `'list'` - default reporter, prints a single line per test;
+   * - `'dot'` - minimal reporter that prints a single character per test run, useful on CI;
+   * - `'line'` - uses a single line for all successfull runs, useful for large test suites;
+   * - `'json'` - outputs a json file with information about the run;
+   * - `'junit'` - outputs an xml file with junit-alike information about the run;
+   * - `'null'` - no reporter, test run will be silent.
+   *
+   * It is possible to pass multiple reporters. A common pattern is using one terminal reporter
+   * like `'line'` or `'list'`, and one file reporter like `'json'` or `'junit'`.
+   */
+  reporter?: 'dot' | 'line' | 'list' | 'junit' | 'json' | 'null' | ReporterDescription[];
+
+  /**
+   * Whether to suppress stdio output from the tests.
+   */
+  quiet?: boolean;
+
+  /**
+   * Shard tests and execute only the selected shard.
+   * Specify in the one-based form `{ total: 5, current: 2 }`.
+   */
+  shard?: Shard;
+
+  /**
+   * Whether to update expected snapshots with the actual results produced by the test run.
+   */
+  updateSnapshots?: UpdateSnapshots;
+
+  /**
+   * The maximum number of concurrent worker processes to use for parallelizing tests.
+   */
+  workers?: number;
+}
+
+/**
+ * Testing configuration.
+ */
+export interface Config<TestArgs = {}, WorkerArgs = {}> extends ConfigBase, Project<TestArgs, WorkerArgs> {
+  /**
+   * Projects specify test files that are executed with a specific configuration.
+   */
+  projects?: Project<TestArgs, WorkerArgs>[];
+}
+
+export interface FullConfig {
+  forbidOnly: boolean;
+  globalSetup: string | null;
+  globalTeardown: string | null;
+  globalTimeout: number;
+  grep: RegExp | RegExp[];
+  maxFailures: number;
+  preserveOutput: PreserveOutput;
+  projects: FullProject[];
+  reporter: ReporterDescription[];
+  rootDir: string;
+  quiet: boolean;
+  shard: Shard;
+  updateSnapshots: UpdateSnapshots;
+  workers: number;
+}
+
+export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
+
+/**
+ * Information common for all tests run in the same worker process.
+ */
+export interface WorkerInfo {
+  /**
+   * Testing configuration.
+   */
+  config: FullConfig;
+
+  /**
+   * Specific project configuration for this worker.
+   * Different projects are always run in separate processes.
+   */
+  project: FullProject;
+
+  /**
+   * Unique worker index. Also available as `process.env.TEST_WORKER_INDEX`.
+   */
+  workerIndex: number;
+}
+
+/**
+ * Information about a particular test run.
+ */
+export interface TestInfo extends WorkerInfo {
+  /**
+   * Test title as passed to `test('my test title', testFunction)`.
+   */
+  title: string;
+
+  /**
+   * Path to the file where test is declared.
+   */
+  file: string;
+
+  /**
+   * Line number in the test file where the test is declared.
+   */
+  line: number;
+
+  /**
+   * Column number in the test file where the test is declared.
+   */
+  column: number;
+
+  /**
+   * The test function as passed to `test('my test title', testFunction)`.
+   */
+  fn: Function;
+
+  /**
+   * Call this method to skip the current test.
+   */
+  skip(): void;
+  skip(condition: boolean): void;
+  skip(condition: boolean, description: string): void;
+
+  /**
+   * Call this method to mark the current test as "needs to be fixed". The test will not be run.
+   */
+  fixme(): void;
+  fixme(condition: boolean): void;
+  fixme(condition: boolean, description: string): void;
+
+  /**
+   * Call this method to mark the current test as "expected to fail". The test will be run and must fail.
+   */
+  fail(): void;
+  fail(condition: boolean): void;
+  fail(condition: boolean, description: string): void;
+
+  /**
+   * Call this method to mark the current test as slow. The default timeout will be trippled.
+   */
+  slow(): void;
+  slow(condition: boolean): void;
+  slow(condition: boolean, description: string): void;
+
+  /**
+   * Call this method to set a custom timeout for the current test.
+   */
+  setTimeout(timeout: number): void;
+
+  /**
+   * The expected status for the test:
+   * - `'passed'` for most tests;
+   * - `'failed'` for tests marked with `test.fail()`;
+   * - `'skipped'` for tests marked with `test.skip()` or `test.fixme()`.
+   */
+  expectedStatus: TestStatus;
+
+  /**
+   * Timeout in milliseconds for this test.
+   */
+  timeout: number;
+
+  /**
+   * Annotations collected for this test.
+   */
+  annotations: { type: string, description?: string }[];
+
+  /**
+   * When tests are run multiple times, each run gets a unique `repeatEachIndex`.
+   */
+  repeatEachIndex: number;
+
+  /**
+   * When the test is retried after a failure, `retry` indicates the attempt number.
+   * Zero for the first (non-retry) run.
+   *
+   * The maximum number of retries is configurable with `retries` field in the config.
+   */
+  retry: number;
+
+  /**
+   * The number of milliseconds this test took to finish.
+   * Only available after the test has finished.
+   */
+  duration: number;
+
+  /**
+   * The result of the run.
+   * Only available after the test has finished.
+   */
+  status?: TestStatus;
+
+  /**
+   * The error thrown by the test if any.
+   * Only available after the test has finished.
+   */
+  error?: any;
+
+  /**
+   * Output written to `process.stdout` or `console.log` from the test.
+   * Only available after the test has finished.
+   */
+  stdout: (string | Buffer)[];
+
+  /**
+   * Output written to `process.stderr` or `console.error` from the test.
+   * Only available after the test has finished.
+   */
+  stderr: (string | Buffer)[];
+
+  /**
+   * Suffix used to differentiate snapshots between multiple test configurations.
+   * For example, if snapshots depend on the platform, you can set `testInfo.snapshotSuffix = process.platform`,
+   * and `expect(value).toMatchSnapshot(snapshotName)` will use different snapshots depending on the platform.
+   */
+  snapshotSuffix: string;
+
+  /**
+   * Absolute path to the output directory for this specific test run.
+   * Each test gets its own directory.
+   */
+  outputDir: string;
+
+  /**
+   * Returns a path to a snapshot file.
+   */
+  snapshotPath: (snapshotName: string) => string;
+
+  /**
+   * Returns a path inside the `outputDir` where the test can safely put a temporary file.
+   * Guarantees that tests running in parallel will not interfere with each other.
+   *
+   * ```js
+   * const file = testInfo.outputPath('temporary-file.txt');
+   * await fs.promises.writeFile(file, 'Put some data to the file', 'utf8');
+   * ```
+   */
+  outputPath: (...pathSegments: string[]) => string;
+}
+
+interface SuiteFunction {
+  (name: string, inner: () => void): void;
+}
+
+interface TestFunction<TestArgs> {
+  (name: string, inner: (args: TestArgs, testInfo: TestInfo) => Promise<void> | void): void;
+}
+
+/**
+ * Call this function to declare a test.
+ *
+ * ```js
+ * test('my test title', async () => {
+ *   // Test code goes here.
+ * });
+ * ```
+ */
+export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue> extends TestFunction<TestArgs & WorkerArgs> {
+  /**
+   * Use `test.only()` instead of `test()` to ignore all other tests and only run this one.
+   * Useful for debugging a particular test.
+   *
+   * ```js
+   * test.only('my test title', async () => {
+   *   // Only this test will run.
+   * });
+   * ```
+   *
+   * All tests marked as `test.only()` will be run, so you can mark multiple of them.
+   */
+  only: TestFunction<TestArgs & WorkerArgs>;
+
+  /**
+   * Declare a block of related tests.
+   *
+   * ```js
+   * test.decribe('my test suite', () => {
+   *   test('one test', async () => {
+   *     // Test code goes here.
+   *   });
+   *
+   *   test('another test', async () => {
+   *     // Test code goes here.
+   *   });
+   * });
+   * ```
+   *
+   * Any `beforeEach`, `afterEach`, `beforeAll` and `afterAll` hooks declared inside the `test.decribe()` block
+   * will only affect the tests from this block.
+   */
+  describe: SuiteFunction & {
+    /**
+     * Use `test.describe.only()` instead of `test.describe()` to ignore all other tests and only run this block.
+     * Useful for debugging a few tests.
+     */
+    only: SuiteFunction;
+  };
+
+  /**
+   * Skip running this test.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.skip();
+   *   // Test code goes here. It will not be executed.
+   * });
+   * ```
+   */
+  skip(): void;
+
+  /**
+   * Skip running this test when `condition` is true.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.skip(process.platform === 'darwin');
+   *   // Test code goes here. It will not be executed on MacOS.
+   * });
+   * ```
+   */
+  skip(condition: boolean): void;
+
+  /**
+   * Skip running this test when `condition` is true.
+   * Put a reason in `description` to easily remember it later.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.skip(process.platform === 'darwin', 'Dependency "foo" is crashing on MacOS');
+   *   // Test code goes here. It will not be executed on MacOS.
+   * });
+   * ```
+   */
+  skip(condition: boolean, description: string): void;
+
+  /**
+   * Skip running tests in the `describe` block based on some condition.
+   *
+   * ```js
+   * test.describe('my tests', () => {
+   *   test.skip(() => process.platform === 'darwin');
+   *
+   *   // Declare tests below - they will not be executed on MacOS.
+   * });
+   * ```
+   */
+  skip(callback: (args: TestArgs & WorkerArgs) => boolean): void;
+
+  /**
+   * Skip running tests in the `describe` block based on some condition.
+   * Put a reason in `description` to easily remember it later.
+   *
+   * ```js
+   * test.describe('my tests', () => {
+   *   test.skip(() => process.platform === 'darwin', 'Dependency "foo" is crashing on MacOS');
+   *
+   *   // Declare tests below - they will not be executed on MacOS.
+   * });
+   * ```
+   */
+  skip(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void;
+
+  /**
+   * Skip running this test, with intention to fix it later.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.fixme();
+   *   // Test code goes here. It will not be executed.
+   * });
+   * ```
+   */
+  fixme(): void;
+
+  /**
+   * Skip running this test when `condition` is true, with intention to fix it later.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.fixme(process.platform === 'darwin');
+   *   // Test code goes here. It will not be executed on MacOS.
+   * });
+   * ```
+   */
+  fixme(condition: boolean): void;
+
+  /**
+   * Skip running this test when `condition` is true, with intention to fix it later.
+   * Put a reason in `description` to easily remember it later.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.fixme(process.platform === 'darwin', 'Dependency "foo" is crashing on MacOS');
+   *   // Test code goes here. It will not be executed on MacOS.
+   * });
+   * ```
+   */
+  fixme(condition: boolean, description: string): void;
+
+  /**
+   * Skip running tests in the `describe` block based on some condition, with intention to fix it later.
+   *
+   * ```js
+   * test.describe('my tests', () => {
+   *   test.fixme(() => process.platform === 'darwin');
+   *
+   *   // Declare tests below - they will not be executed on MacOS.
+   * });
+   * ```
+   */
+  fixme(callback: (args: TestArgs & WorkerArgs) => boolean): void;
+
+  /**
+   * Skip running tests in the `describe` block based on some condition, with intention to fix it later.
+   * Put a reason in `description` to easily remember it later.
+   *
+   * ```js
+   * test.describe('my tests', () => {
+   *   test.fixme(() => process.platform === 'darwin', 'Dependency "foo" is crashing on MacOS');
+   *
+   *   // Declare tests below - they will not be executed on MacOS.
+   * });
+   * ```
+   */
+  fixme(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void;
+
+  /**
+   * Mark the test as "expected to fail". It will be run and should fail.
+   * When "expected to fail" test acceidentally passes, test runner will exit with an error.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.fail();
+   *   // Test code goes here.
+   * });
+   * ```
+   */
+  fail(): void;
+
+  /**
+   * Mark the test as "expected to fail", when `condition` is true. It will be run and should fail.
+   * When "expected to fail" test acceidentally passes, test runner will exit with an error.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.fail(process.platform === 'darwin');
+   *   // Test code goes here. It should fail on MacOS.
+   * });
+   * ```
+   */
+  fail(condition: boolean): void;
+
+  /**
+   * Mark the test as "expected to fail", when `condition` is true. It will be run and should fail.
+   * When "expected to fail" test acceidentally passes, test runner will exit with an error.
+   * Put a reason in `description` to easily remember it later.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.fail(process.platform === 'darwin', 'Could not find resources - see issue #1234');
+   *   // Test code goes here. It should fail on MacOS.
+   * });
+   * ```
+   */
+  fail(condition: boolean, description: string): void;
+
+  /**
+   * Mark tests in the `describe` block as "expected to fail" based on some condition.
+   * The tests will be run and should fail.
+   * When "expected to fail" test acceidentally passes, test runner will exit with an error.
+   *
+   * ```js
+   * test.describe('my tests', () => {
+   *   test.fail(() => process.platform === 'darwin');
+   *
+   *   // Declare tests below - they should fail on MacOS.
+   * });
+   * ```
+   */
+  fail(callback: (args: TestArgs & WorkerArgs) => boolean): void;
+
+  /**
+   * Mark tests in the `describe` block as "expected to fail" based on some condition.
+   * The tests will be run and should fail.
+   * When "expected to fail" test acceidentally passes, test runner will exit with an error.
+   * Put a reason in `description` to easily remember it later.
+   *
+   * ```js
+   * test.describe('my tests', () => {
+   *   test.fail(() => process.platform === 'darwin', 'Could not find resources - see issue #1234');
+   *
+   *   // Declare tests below - they should fail on MacOS.
+   * });
+   * ```
+   */
+  fail(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void;
+
+  /**
+   * Triples the default timeout for this test.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.slow();
+   *   // Test code goes here.
+   * });
+   * ```
+   */
+  slow(): void;
+
+  /**
+   * Triples the default timeout for this test, when `condition` is true.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.slow(process.platform === 'darwin');
+   *   // Test code goes here. It will be given triple timeout on MacOS.
+   * });
+   * ```
+   */
+  slow(condition: boolean): void;
+
+  /**
+   * Triples the default timeout for this test, when `condition` is true.
+   * Put a reason in `description` to easily remember it later.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   test.slow(process.platform === 'darwin', 'Dependency "foo" is slow on MacOS');
+   *   // Test code goes here. It will be given triple timeout on MacOS.
+   * });
+   * ```
+   */
+  slow(condition: boolean, description: string): void;
+
+  /**
+   * Give all tests in the `describe` block triple timeout, based on some condition.
+   *
+   * ```js
+   * test.describe('my tests', () => {
+   *   test.slow(() => process.platform === 'darwin');
+   *
+   *   // Declare tests below - they will be given triple timeout on MacOS.
+   * });
+   * ```
+   */
+  slow(callback: (args: TestArgs & WorkerArgs) => boolean): void;
+
+  /**
+   * Give all tests in the `describe` block triple timeout, based on some condition.
+   * Put a reason in `description` to easily remember it later.
+   *
+   * ```js
+   * test.describe('my tests', () => {
+   *   test.slow(() => process.platform === 'darwin', 'Dependency "foo" is slow on MacOS');
+   *
+   *   // Declare tests below - they will be given triple timeout on MacOS.
+   * });
+   * ```
+   */
+  slow(callback: (args: TestArgs & WorkerArgs) => boolean, description: string): void;
+
+  /**
+   * Set a custom timeout for the test.
+   *
+   * ```js
+   * test('my test title', async () => {
+   *   // Give this test 20 seconds.
+   *   test.setTimeout(20000);
+   *   // Test code goes here.
+   * });
+   * ```
+   */
+  setTimeout(timeout: number): void;
+
+  /**
+   * Declare a hook that will be run before each test.
+   * It may use all the available fixtures.
+   *
+   * ```js
+   * test.beforeEach(async ({ fixture }, testInfo) => {
+   *   // Do some work here.
+   * });
+   * ```
+   *
+   * When called inside a `test.describe()` block, the hook only applies to the tests from the block.
+   */
+  beforeEach(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<any> | any): void;
+
+  /**
+   * Declare a hook that will be run after each test.
+   * It may use all the available fixtures.
+   *
+   * ```js
+   * test.afterEach(async ({ fixture }, testInfo) => {
+   *   // Do some work here.
+   * });
+   * ```
+   *
+   * When called inside a `test.describe()` block, the hook only applies to the tests from the block.
+   */
+  afterEach(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<any> | any): void;
+
+  /**
+   * Declare a hook that will be run once before all tests in the file.
+   * It may use all worker-scoped fixtures.
+   *
+   * ```js
+   * test.beforeAll(async ({ workerFixture }, workerInfo) => {
+   *   // Do some work here.
+   * });
+   * ```
+   *
+   * When called inside a `test.describe()` block, the hook only applies to the tests from the block.
+   */
+  beforeAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise<any> | any): void;
+
+  /**
+   * Declare a hook that will be run once after all tests in the file.
+   * It may use all worker-scoped fixtures.
+   *
+   * ```js
+   * test.afterAll(async ({ workerFixture }, workerInfo) => {
+   *   // Do some work here.
+   * });
+   * ```
+   *
+   * When called inside a `test.describe()` block, the hook only applies to the tests from the block.
+   */
+  afterAll(inner: (args: WorkerArgs, workerInfo: WorkerInfo) => Promise<any> | any): void;
+
+  /**
+   * Declare fixtures/options to be used for tests in this file.
+   *
+   * ```js
+   * test.use({ myOption: 'foo' });
+   *
+   * test('my test title', async ({ myFixtureThatUsesMyOption }) => {
+   *   // Test code goes here.
+   * });
+   * ```
+   *
+   * When called inside a `test.describe()` block, fixtures/options only apply to the tests from the block.
+   */
+  use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void;
+
+  /**
+   * Use `test.expect(value).toBe(expected)` to assert something in the test.
+   * See [expect library](https://jestjs.io/docs/expect) documentation for more details.
+   */
+  expect: Expect;
+
+  declare<T extends KeyValue = {}, W extends KeyValue = {}>(): TestType<TestArgs & T, WorkerArgs & W>;
+
+  /**
+   * Extend the test with fixtures. These fixtures will be invoked for test when needed,
+   * can perform setup/teardown and provide a resource to the test.
+   *
+   * ```ts
+   * import base from 'folio';
+   * import rimraf from 'rimraf';
+   *
+   * const test = base.extend<{ dirCount: number, dirs: string[] }>({
+   *   // Define an option that can be configured in tests with `test.use()`.
+   *   // Provide a default value.
+   *   dirCount: 1,
+   *
+   *   // Define a fixture that provides some useful functionality to the test.
+   *   // In this example, it will create some temporary directories.
+   *   dirs: async ({ dirCount }, use, testInfo) => {
+   *     // Our fixture uses the "dirCount" option that can be configured by the test.
+   *     const dirs = [];
+   *     for (let i = 0; i < dirCount; i++) {
+   *       // Create an isolated directory.
+   *       const dir = testInfo.outputPath('dir-' + i);
+   *       await fs.promises.mkdir(dir, { recursive: true });
+   *       dirs.push(dir);
+   *     }
+   *
+   *     // Use the list of directories in the test.
+   *     await use(dirs);
+   *
+   *     // Cleanup if needed.
+   *     for (const dir of dirs)
+   *       await new Promise(done => rimraf(dir, done));
+   *   },
+   * });
+   *
+   *
+   * // Tests in this file need two temporary directories.
+   * test.use({ dirCount: 2 });
+   *
+   * test('my test title', async ({ dirs }) => {
+   *   // Test code goes here.
+   *   // It can use "dirs" right away - the fixture has already run and created two temporary directories.
+   * });
+   * ```
+   */
+  extend<T, W extends KeyValue = {}>(fixtures: Fixtures<T, W, TestArgs, WorkerArgs>): TestType<TestArgs & T, WorkerArgs & W>;
+}
+
+type KeyValue = { [key: string]: any };
+export type TestFixture<R, Args extends KeyValue> = (args: Args, use: (r: R) => Promise<void>, testInfo: TestInfo) => any;
+export type WorkerFixture<R, Args extends KeyValue> = (args: Args, use: (r: R) => Promise<void>, workerInfo: WorkerInfo) => any;
+type TestFixtureValue<R, Args> = R | TestFixture<R, Args>;
+type WorkerFixtureValue<R, Args> = R | WorkerFixture<R, Args>;
+export type Fixtures<T extends KeyValue = {}, W extends KeyValue = {}, PT extends KeyValue = {}, PW extends KeyValue = {}> = {
+  [K in keyof PW]?: WorkerFixtureValue<PW[K], W & PW>;
+} & {
+  [K in keyof PT]?: TestFixtureValue<PT[K], T & W & PT & PW>;
+} & {
+  [K in keyof W]?: [WorkerFixtureValue<W[K], W & PW>, { scope: 'worker', auto?: boolean }];
+} & {
+  [K in keyof T]?: TestFixtureValue<T[K], T & W & PT & PW> | [TestFixtureValue<T[K], T & W & PT & PW>, { scope?: 'test', auto?: boolean }];
+};
+
+export type Location = { file: string, line: number, column: number };
+export type FixturesWithLocation = {
+  fixtures: Fixtures;
+  location: Location;
+};

utils/check_deps.js

@@ -163,6 +163,9 @@ DEPS['src/server/trace/common/'] = ['src/server/snapshot/', ...DEPS['src/server/
 DEPS['src/server/trace/recorder/'] = ['src/server/trace/common/', ...DEPS['src/server/trace/common/']];
 DEPS['src/server/trace/viewer/'] = ['src/server/trace/common/', ...DEPS['src/server/trace/common/']];
 
+// No dependencies for test runner.
+DEPS['src/test/'] = ['src/test/**'];
+
 checkDeps().catch(e => {
   console.error(e && e.stack ? e.stack : e);
   process.exit(1);