feat(test): introduce npx playwright test (#6816)

packages/build_package.js

@@ -42,6 +42,12 @@ const PACKAGES = {
     browsers: [],
     files: PLAYWRIGHT_CORE_FILES,
   },
+  'playwright-test': {
+    description: 'Playwright Test Runner',
+    browsers: ['chromium', 'firefox', 'webkit', 'ffmpeg'],
+    files: PLAYWRIGHT_CORE_FILES,
+    name: '@playwright/test',
+  },
   'playwright-webkit': {
     description: 'A high-level API to automate WebKit',
     browsers: ['webkit'],
@@ -115,9 +121,12 @@ if (!args.some(arg => arg === '--no-cleanup')) {
 
   // 4. Generate package.json
   const pwInternalJSON = require(path.join(ROOT_PATH, 'package.json'));
+  const dependencies = { ...pwInternalJSON.dependencies };
+  if (packageName === 'playwright-test')
+    dependencies.folio = pwInternalJSON.devDependencies.folio;
   await writeToPackage('package.json', JSON.stringify({
-    name: packageName,
-    version: package.version || pwInternalJSON.version,
+    name: package.name || packageName,
+    version: pwInternalJSON.version,
     description: package.description,
     repository: pwInternalJSON.repository,
     engines: pwInternalJSON.engines,
@@ -126,9 +135,6 @@ if (!args.some(arg => arg === '--no-cleanup')) {
     bin: {
       playwright: './lib/cli/cli.js',
     },
-    engines: {
-      node: '>=12',
-    },
     exports: {
       // Root import: we have a wrapper ES Module to support the following syntax.
       // const { chromium } = require('playwright');
@@ -145,7 +151,7 @@ if (!args.some(arg => arg === '--no-cleanup')) {
     },
     author: pwInternalJSON.author,
     license: pwInternalJSON.license,
-    dependencies: pwInternalJSON.dependencies
+    dependencies,
   }, null, 2));
 
   // 5. Generate browsers.json

packages/installation-tests/esm-playwright-test.mjs

@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { chromium, firefox, webkit, selectors, devices, errors } from '@playwright/test';
+import playwright from '@playwright/test';
+import errorsFile from '@playwright/test/lib/utils/errors.js';
+
+import testESM from './esm.mjs';
+testESM({ chromium, firefox, webkit, selectors, devices, errors, playwright, errorsFile }, [chromium, firefox, webkit]);

packages/installation-tests/installation-tests.sh

@@ -27,6 +27,8 @@ PLAYWRIGHT_WEBKIT_TGZ="$(node ${PACKAGE_BUILDER} playwright-webkit ./playwright-
 echo "playwright-webkit built"
 PLAYWRIGHT_FIREFOX_TGZ="$(node ${PACKAGE_BUILDER} playwright-firefox ./playwright-firefox.tgz)"
 echo "playwright-firefox built"
+PLAYWRIGHT_TEST_TGZ="$(node ${PACKAGE_BUILDER} playwright-test ./playwright-test.tgz)"
+echo "playwright-test built"
 
 SCRIPTS_PATH="$(pwd -P)/.."
 TEST_ROOT="/tmp/playwright-installation-tests"
@@ -45,12 +47,16 @@ function copy_test_scripts {
   cp "${SCRIPTS_PATH}/esm-playwright-chromium.mjs" .
   cp "${SCRIPTS_PATH}/esm-playwright-firefox.mjs" .
   cp "${SCRIPTS_PATH}/esm-playwright-webkit.mjs" .
+  cp "${SCRIPTS_PATH}/esm-playwright-test.mjs" .
   cp "${SCRIPTS_PATH}/sanity-electron.js" .
   cp "${SCRIPTS_PATH}/electron-app.js" .
   cp "${SCRIPTS_PATH}/driver-client.js" .
+  cp "${SCRIPTS_PATH}/sample.spec.js" .
+  cp "${SCRIPTS_PATH}/read-json-report.js" .
 }
 
 function run_tests {
+  test_playwright_test_should_work
   test_screencast
   test_typescript_types
   test_playwright_global_installation_subsequent_installs
@@ -252,6 +258,12 @@ function test_playwright_should_work {
     node esm-playwright.mjs
   fi
 
+  echo "Running playwright test"
+  if npx playwright test -c .; then
+    echo "ERROR: should not be able to run tests with just playwright package"
+    exit 1
+  fi
+
   echo "${FUNCNAME[0]} success"
 }
 
@@ -555,6 +567,37 @@ function test_playwright_driver_should_work {
   echo "${FUNCNAME[0]} success"
 }
 
+function test_playwright_test_should_work {
+  initialize_test "${FUNCNAME[0]}"
+
+  npm install ${PLAYWRIGHT_TEST_TGZ}
+  copy_test_scripts
+
+  echo "Running playwright test without install"
+  if npx playwright test -c .; then
+    echo "ERROR: should not be able to run tests without installing browsers"
+    exit 1
+  fi
+
+  echo "Running playwright install"
+  PLAYWRIGHT_BROWSERS_PATH="0" npx playwright install
+
+  echo "Running playwright test"
+  PLAYWRIGHT_JSON_OUTPUT_NAME=report.json PLAYWRIGHT_BROWSERS_PATH="0" npx playwright test -c . --browser=all --reporter=list,json
+
+  echo "Checking the report"
+  node ./read-json-report.js ./report.json
+
+  echo "Running sanity.js"
+  node sanity.js "@playwright/test"
+  if [[ "${NODE_VERSION}" == *"v14."* ]]; then
+    echo "Running esm.js"
+    node esm-playwright-test.mjs
+  fi
+
+  echo "${FUNCNAME[0]} success"
+}
+
 function initialize_test {
   cd ${TEST_ROOT}
   local TEST_NAME="./$1"

packages/installation-tests/read-json-report.js

@@ -0,0 +1,18 @@
+const report = require(process.argv[2]);
+if (report.suites[0].specs[0].title !== 'sample test') {
+  console.log(`Wrong spec title`);
+  process.exit(1);
+}
+const projects = report.suites[0].specs[0].tests.map(t => t.projectName).sort();
+if (projects.length !== 3 || projects[0] !== 'chromium' || projects[1] !== 'firefox' || projects[2] !== 'webkit') {
+  console.log(`Wrong browsers`);
+  process.exit(1);
+}
+for (const test of report.suites[0].specs[0].tests) {
+  if (test.results[0].status !== 'passed') {
+    console.log(`Test did not pass`);
+    process.exit(1);
+  }
+}
+console.log('Report check SUCCESS');
+process.exit(0);

packages/installation-tests/sample.spec.js

@@ -0,0 +1,6 @@
+const { test, expect } = require('@playwright/test');
+
+test('sample test', async ({ page }) => {
+  await page.setContent(`<div>hello</div><span>world</span>`);
+  expect(await page.textContent('span')).toBe('world');
+});

packages/installation-tests/sanity.js

@@ -20,6 +20,7 @@ let success = {
   'playwright-chromium': ['chromium'],
   'playwright-firefox': ['firefox'],
   'playwright-webkit': ['webkit'],
+  '@playwright/test': ['chromium', 'firefox', 'webkit'],
 }[requireName];
 if (process.argv[3] === 'none')
   success = [];

packages/playwright-test/README.md

@@ -0,0 +1,3 @@
+# @playwright/test
+
+This package contains [Playwright Test Runner](https://playwright.dev/docs/test-intro).

packages/playwright-test/index.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as types from './types/types';
+
+export * from './types/types';
+export const chromium: types.BrowserType;
+export const firefox: types.BrowserType;
+export const webkit: types.BrowserType;
+export const _electron: types.Electron;
+export const _android: types.Android;
+export * from './types/test';

packages/playwright-test/index.js

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module.exports = {
+  ...require('./lib/inprocess'),
+  ...require('./lib/cli/fixtures')
+};

packages/playwright-test/index.mjs

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import playwright from './index.js';
+
+export const chromium = playwright.chromium;
+export const firefox = playwright.firefox;
+export const webkit = playwright.webkit;
+export const selectors = playwright.selectors;
+export const devices = playwright.devices;
+export const errors = playwright.errors;
+export const _electron = playwright._electron;
+export const _android = playwright._android;
+export const test = playwright.test;
+export default playwright;

packages/playwright-test/install.js

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Explicitly empty install.js to avoid touching browser registry at all.

src/cli/cli.ts

@@ -35,6 +35,7 @@ import { BrowserContextOptions, LaunchOptions } from '../client/types';
 import { spawn } from 'child_process';
 import { installDeps } from '../install/installDeps';
 import { allBrowserNames, BrowserName } from '../utils/registry';
+import { addTestCommand } from './testRunner';
 import * as utils from '../utils/utils';
 
 const SCRIPTS_DIRECTORY = path.join(__dirname, '..', '..', 'bin');
@@ -225,6 +226,9 @@ program
       console.log('  $ show-trace trace/directory');
     });
 
+if (!process.env.PW_CLI_TARGET_LANG)
+  addTestCommand(program);
+
 if (process.argv[2] === 'run-driver')
   runDriver();
 else if (process.argv[2] === 'run-server')

src/cli/fixtures.ts

@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import * as util from 'util';
+import * as folio from 'folio';
+import type { LaunchOptions, BrowserContextOptions, Page } from '../../types/types';
+import type { PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions } from '../../types/test';
+
+export * from 'folio';
+export const test = folio.test.extend<PlaywrightTestArgs & PlaywrightTestOptions, PlaywrightWorkerArgs & PlaywrightWorkerOptions>({
+  browserName: [ 'chromium', { scope: 'worker' } ],
+  playwright: [ require('../inprocess'), { scope: 'worker' } ],
+  headless: [ undefined, { scope: 'worker' } ],
+  channel: [ undefined, { scope: 'worker' } ],
+  launchOptions: [ {}, { scope: 'worker' } ],
+
+  browser: [ async ({ playwright, browserName, headless, channel, launchOptions }, use) => {
+    if (!['chromium', 'firefox', 'webkit'].includes(browserName))
+      throw new Error(`Unexpected browserName "${browserName}", must be one of "chromium", "firefox" or "webkit"`);
+    const options: LaunchOptions = {
+      handleSIGINT: false,
+      ...launchOptions,
+    };
+    if (headless !== undefined)
+      options.headless = headless;
+    if (channel !== undefined)
+      options.channel = channel;
+    const browser = await playwright[browserName].launch(options);
+    await use(browser);
+    await browser.close();
+  }, { scope: 'worker' } ],
+
+  screenshot: 'off',
+  video: 'off',
+  acceptDownloads: undefined,
+  bypassCSP: undefined,
+  colorScheme: undefined,
+  deviceScaleFactor: undefined,
+  extraHTTPHeaders: undefined,
+  geolocation: undefined,
+  hasTouch: undefined,
+  httpCredentials: undefined,
+  ignoreHTTPSErrors: undefined,
+  isMobile: undefined,
+  javaScriptEnabled: undefined,
+  locale: undefined,
+  offline: undefined,
+  permissions: undefined,
+  proxy: undefined,
+  storageState: undefined,
+  timezoneId: undefined,
+  userAgent: undefined,
+  viewport: undefined,
+  contextOptions: {},
+
+  context: async ({ browserName, browser, screenshot, video, acceptDownloads, bypassCSP, colorScheme, deviceScaleFactor, extraHTTPHeaders, hasTouch, geolocation, httpCredentials, ignoreHTTPSErrors, isMobile, javaScriptEnabled, locale, offline, permissions, proxy, storageState, viewport, timezoneId, userAgent, contextOptions }, use, testInfo) => {
+    testInfo.snapshotSuffix = browserName + '-' + process.platform;
+    if (process.env.PWDEBUG)
+      testInfo.setTimeout(0);
+
+    const recordVideo = video === 'on' || video === 'retain-on-failure' ||
+      (video === 'retry-with-video' && !!testInfo.retry);
+    const options: BrowserContextOptions = {
+      recordVideo: recordVideo ? { dir: testInfo.outputPath('') } : undefined,
+      ...contextOptions,
+    };
+    if (acceptDownloads !== undefined)
+      options.acceptDownloads = acceptDownloads;
+    if (bypassCSP !== undefined)
+      options.bypassCSP = bypassCSP;
+    if (colorScheme !== undefined)
+      options.colorScheme = colorScheme;
+    if (deviceScaleFactor !== undefined)
+      options.deviceScaleFactor = deviceScaleFactor;
+    if (extraHTTPHeaders !== undefined)
+      options.extraHTTPHeaders = extraHTTPHeaders;
+    if (geolocation !== undefined)
+      options.geolocation = geolocation;
+    if (hasTouch !== undefined)
+      options.hasTouch = hasTouch;
+    if (httpCredentials !== undefined)
+      options.httpCredentials = httpCredentials;
+    if (ignoreHTTPSErrors !== undefined)
+      options.ignoreHTTPSErrors = ignoreHTTPSErrors;
+    if (isMobile !== undefined)
+      options.isMobile = isMobile;
+    if (javaScriptEnabled !== undefined)
+      options.javaScriptEnabled = javaScriptEnabled;
+    if (locale !== undefined)
+      options.locale = locale;
+    if (offline !== undefined)
+      options.offline = offline;
+    if (permissions !== undefined)
+      options.permissions = permissions;
+    if (proxy !== undefined)
+      options.proxy = proxy;
+    if (storageState !== undefined)
+      options.storageState = storageState;
+    if (timezoneId !== undefined)
+      options.timezoneId = timezoneId;
+    if (userAgent !== undefined)
+      options.userAgent = userAgent;
+    if (viewport !== undefined)
+      options.viewport = viewport;
+
+    const context = await browser.newContext(options);
+    const allPages: Page[] = [];
+    context.on('page', page => allPages.push(page));
+
+    await use(context);
+
+    const testFailed = testInfo.status !== testInfo.expectedStatus;
+    if (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed)) {
+      await Promise.all(allPages.map((page, index) => {
+        const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${++index}.png`);
+        return page.screenshot({ timeout: 5000, path: screenshotPath }).catch(e => {});
+      }));
+    }
+    await context.close();
+
+    const deleteVideos = video === 'retain-on-failure' && !testFailed;
+    if (deleteVideos) {
+      await Promise.all(allPages.map(async page => {
+        const video = page.video();
+        if (!video)
+          return;
+        const videoPath = await video.path();
+        await util.promisify(fs.unlink)(videoPath).catch(e => {});
+      }));
+    }
+  },
+
+  page: async ({ context }, use) => {
+    await use(await context.newPage());
+  },
+});
+export default test;

src/cli/testRunner.ts

@@ -0,0 +1,188 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* eslint-disable no-console */
+
+import * as commander from 'commander';
+import * as fs from 'fs';
+import * as path from 'path';
+import type { Config } from 'folio';
+
+type RunnerType = typeof import('folio/out/runner').Runner;
+
+const defaultTimeout = 30000;
+const defaultReporter = process.env.CI ? 'dot' : 'list';
+const builtinReporters = ['list', 'line', 'dot', 'json', 'junit', 'null'];
+const tsConfig = 'playwright.config.ts';
+const jsConfig = 'playwright.config.js';
+const defaultConfig: Config = {
+  preserveOutput: process.env.CI ? 'failures-only' : 'always',
+  reporter: [defaultReporter],
+  timeout: defaultTimeout,
+  updateSnapshots: process.env.CI ? 'none' : 'missing',
+  workers: Math.ceil(require('os').cpus().length / 2),
+};
+
+export function addTestCommand(program: commander.CommanderStatic) {
+  let Runner: RunnerType;
+  try {
+    Runner = require('folio/out/runner').Runner as RunnerType;
+  } catch (e) {
+    addStubTestCommand(program);
+    return;
+  }
+
+  const command = program.command('test [test-filter...]');
+  command.description('Run tests with Playwright Test');
+  command.option('--browser <browser>', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`);
+  command.option('--headed', `Run tests in headed browsers (default: headless)`);
+  command.option('-c, --config <file>', `Configuration file, or a test directory with optional "${tsConfig}"/"${jsConfig}"`);
+  command.option('--forbid-only', `Fail if test.only is called (default: false)`);
+  command.option('-g, --grep <grep>', `Only run tests matching this regular expression (default: ".*")`);
+  command.option('--global-timeout <timeout>', `Maximum time this test suite can run in milliseconds (default: unlimited)`);
+  command.option('-j, --workers <workers>', `Number of concurrent workers, use 1 to run in a single worker (default: number of CPU cores / 2)`);
+  command.option('--list', `Collect all the tests and report them, but do not run`);
+  command.option('--max-failures <N>', `Stop after the first N failures`);
+  command.option('--output <dir>', `Folder for output artifacts (default: "test-results")`);
+  command.option('--quiet', `Suppress stdio`);
+  command.option('--repeat-each <N>', `Run each test N times (default: 1)`);
+  command.option('--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtinReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`);
+  command.option('--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`);
+  command.option('--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`);
+  command.option('--project <project-name>', `Only run tests from the specified project (default: run all projects)`);
+  command.option('--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`);
+  command.option('-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`);
+  command.option('-x', `Stop after the first failure`);
+  command.action(async (args, opts) => {
+    try {
+      await runTests(Runner, args, opts);
+    } catch (e) {
+      console.error(e.toString());
+      process.exit(1);
+    }
+  });
+  command.on('--help', () => {
+    console.log('');
+    console.log('Arguments [test-filter...]:');
+    console.log('  Pass arguments to filter test files. Each argument is treated as a regular expression.');
+    console.log('');
+    console.log('Examples:');
+    console.log('  $ test my.spec.ts');
+    console.log('  $ test -c tests/ --headed');
+    console.log('  $ test --browser=webkit');
+  });
+}
+
+async function runTests(Runner: RunnerType, args: string[], opts: { [key: string]: any }) {
+  if (opts.browser) {
+    const browserOpt = opts.browser.toLowerCase();
+    if (!['all', 'chromium', 'firefox', 'webkit'].includes(browserOpt))
+      throw new Error(`Unsupported browser "${opts.browser}", must be one of "all", "chromium", "firefox" or "webkit"`);
+    const browserNames = browserOpt === 'all' ? ['chromium', 'firefox', 'webkit'] : [browserOpt];
+    defaultConfig.projects = browserNames.map(browserName => {
+      return {
+        name: browserName,
+        use: { browserName },
+      };
+    });
+  }
+
+  const overrides = overridesFromOptions(opts);
+  if (opts.headed)
+    overrides.use = { headless: false };
+  const runner = new Runner(defaultConfig, overrides);
+
+  function loadConfig(configFile: string) {
+    if (fs.existsSync(configFile)) {
+      if (process.stdout.isTTY)
+        console.log(`Using config at ` + configFile);
+      const loadedConfig = runner.loadConfigFile(configFile);
+      if (('projects' in loadedConfig) && opts.browser)
+        throw new Error(`Cannot use --browser option when configuration file defines projects. Specify browserName in the projects instead.`);
+      return true;
+    }
+    return false;
+  }
+
+  if (opts.config) {
+    const configFile = path.resolve(process.cwd(), opts.config);
+    if (!fs.existsSync(configFile))
+      throw new Error(`${opts.config} does not exist`);
+    if (fs.statSync(configFile).isDirectory()) {
+      // When passed a directory, look for a config file inside.
+      if (!loadConfig(path.join(configFile, tsConfig)) && !loadConfig(path.join(configFile, jsConfig))) {
+        // If there is no config, assume this as a root testing directory.
+        runner.loadEmptyConfig(configFile);
+      }
+    } else {
+      // When passed a file, it must be a config file.
+      loadConfig(configFile);
+    }
+  } else if (!loadConfig(path.resolve(process.cwd(), tsConfig)) && !loadConfig(path.resolve(process.cwd(), jsConfig))) {
+    // No --config option, let's look for the config file in the current directory.
+    // If not, do not assume that current directory is a root testing directory, to avoid scanning the world.
+    throw new Error(`Configuration file not found. Run "npx playwright test --help" for more information.`);
+  }
+
+  process.env.FOLIO_JUNIT_OUTPUT_NAME = process.env.PLAYWRIGHT_JUNIT_OUTPUT_NAME;
+  process.env.FOLIO_JUNIT_SUITE_ID = process.env.PLAYWRIGHT_JUNIT_SUITE_ID;
+  process.env.FOLIO_JUNIT_SUITE_NAME = process.env.PLAYWRIGHT_JUNIT_SUITE_NAME;
+  process.env.FOLIO_JSON_OUTPUT_NAME = process.env.PLAYWRIGHT_JSON_OUTPUT_NAME;
+
+  const result = await runner.run(!!opts.list, args.map(forceRegExp), opts.project || undefined);
+  if (result === 'sigint')
+    process.exit(130);
+  process.exit(result === 'passed' ? 0 : 1);
+}
+
+function forceRegExp(pattern: string): RegExp {
+  const match = pattern.match(/^\/(.*)\/([gi]*)$/);
+  if (match)
+    return new RegExp(match[1], match[2]);
+  return new RegExp(pattern, 'g');
+}
+
+function overridesFromOptions(options: { [key: string]: any }): Config {
+  const isDebuggerAttached = !!require('inspector').url();
+  const shardPair = options.shard ? options.shard.split('/').map((t: string) => parseInt(t, 10)) : undefined;
+  return {
+    forbidOnly: options.forbidOnly ? true : undefined,
+    globalTimeout: isDebuggerAttached ? 0 : (options.globalTimeout ? parseInt(options.globalTimeout, 10) : undefined),
+    grep: options.grep ? forceRegExp(options.grep) : undefined,
+    maxFailures: options.x ? 1 : (options.maxFailures ? parseInt(options.maxFailures, 10) : undefined),
+    outputDir: options.output ? path.resolve(process.cwd(), options.output) : undefined,
+    quiet: options.quiet ? options.quiet : undefined,
+    repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined,
+    retries: options.retries ? parseInt(options.retries, 10) : undefined,
+    reporter: (options.reporter && options.reporter.length) ? options.reporter.split(',').map((r: string) => {
+      return builtinReporters.includes(r) ? r : { require: r };
+    }) : undefined,
+    shard: shardPair ? { current: shardPair[0] - 1, total: shardPair[1] } : undefined,
+    timeout: isDebuggerAttached ? 0 : (options.timeout ? parseInt(options.timeout, 10) : undefined),
+    updateSnapshots: options.updateSnapshots ? 'all' as const : undefined,
+    workers: options.workers ? parseInt(options.workers, 10) : undefined,
+  };
+}
+
+function addStubTestCommand(program: commander.CommanderStatic) {
+  const command = program.command('test');
+  command.description('Run tests with Playwright Test. Available in @playwright/test package.');
+  command.action(async (args, opts) => {
+    console.error('Please install @playwright/test package to use Playwright Test.');
+    console.error('  npm install -D @playwright/test');
+    process.exit(1);
+  });
+}

types/test.d.ts

@@ -0,0 +1,290 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials } from './types';
+import type { Project, Config } from 'folio';
+
+/**
+ * The name of the browser supported by Playwright.
+ */
+type BrowserName = 'chromium' | 'firefox' | 'webkit';
+
+/**
+ * Browser channel name. Used to run tests in different browser flavors,
+ * for example Google Chrome Beta, or Microsoft Edge Stable.
+ * @see BrowserContextOptions
+ */
+type BrowserChannel = Exclude<LaunchOptions['channel'], undefined>;
+
+/**
+ * Emulates `'prefers-colors-scheme'` media feature,
+ * supported values are `'light'`, `'dark'`, `'no-preference'`.
+ * @see BrowserContextOptions
+ */
+type ColorScheme = Exclude<BrowserContextOptions['colorScheme'], undefined>;
+
+/**
+ * An object containing additional HTTP headers to be sent with every request. All header values must be strings.
+ * @see BrowserContextOptions
+ */
+type ExtraHTTPHeaders = Exclude<BrowserContextOptions['extraHTTPHeaders'], undefined>;
+
+/**
+ * Proxy settings available for all tests, or individually per test.
+ * @see BrowserContextOptions
+ */
+type Proxy = Exclude<BrowserContextOptions['proxy'], undefined>;
+
+/**
+ * Storage state for the test.
+ * @see BrowserContextOptions
+ */
+type StorageState = Exclude<BrowserContextOptions['storageState'], undefined>;
+
+/**
+ * Options available to configure browser launch.
+ *   - Set options in config:
+ *   ```js
+ *     use: { browserName: 'webkit' }
+ *   ```
+ *   - Set options in test file:
+ *   ```js
+ *     test.use({ browserName: 'webkit' })
+ *   ```
+ *
+ * Available as arguments to the test function and all hooks (beforeEach, afterEach, beforeAll, afterAll).
+ */
+export type PlaywrightWorkerOptions = {
+  /**
+   * Name of the browser (`chromium`, `firefox`, `webkit`) that runs tests.
+   */
+  browserName: BrowserName;
+
+  /**
+   * Whether to run browser in headless mode. Takes priority over `launchOptions`.
+   * @see LaunchOptions
+   */
+  headless: boolean | undefined;
+
+  /**
+   * Browser distribution channel. Takes priority over `launchOptions`.
+   * @see LaunchOptions
+   */
+  channel: BrowserChannel | undefined;
+
+  /**
+   * Options used to launch the browser. Other options above (e.g. `headless`) take priority.
+   * @see LaunchOptions
+   */
+  launchOptions: LaunchOptions;
+};
+
+/**
+ * Options available to configure each test.
+ *   - Set options in config:
+ *   ```js
+ *     use: { video: 'on' }
+ *   ```
+ *   - Set options in test file:
+ *   ```js
+ *     test.use({ video: 'on' })
+ *   ```
+ *
+ * Available as arguments to the test function and beforeEach/afterEach hooks.
+ */
+export type PlaywrightTestOptions = {
+  /**
+   * Whether to capture a screenshot after each test, off by default.
+   * - `off`: Do not capture screenshots.
+   * - `on`: Capture screenshot after each test.
+   * - `only-on-failure`: Capture screenshot after each test failure.
+   */
+  screenshot: 'off' | 'on' | 'only-on-failure';
+
+  /**
+  * Whether to record video for each test, off by default.
+  * - `off`: Do not record video.
+  * - `on`: Record video for each test.
+  * - `retain-on-failure`: Record video for each test, but remove all videos from successful test runs.
+  * - `retry-with-video`: Record video only when retrying a test.
+  */
+  video: 'off' | 'on' | 'retain-on-failure' | 'retry-with-video';
+
+  /**
+   * Whether to automatically download all the attachments. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  acceptDownloads: boolean | undefined;
+
+  /**
+   * Toggles bypassing page's Content-Security-Policy. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  bypassCSP: boolean | undefined;
+
+  /**
+   * Emulates `'prefers-colors-scheme'` media feature, supported values are `'light'`, `'dark'`, `'no-preference'`.
+   * @see BrowserContextOptions
+   */
+  colorScheme: ColorScheme | undefined;
+
+  /**
+   * Specify device scale factor (can be thought of as dpr). Defaults to `1`.
+   * @see BrowserContextOptions
+   */
+  deviceScaleFactor: number | undefined;
+
+  /**
+   * An object containing additional HTTP headers to be sent with every request. All header values must be strings.
+   * @see BrowserContextOptions
+   */
+  extraHTTPHeaders: ExtraHTTPHeaders | undefined;
+
+  /**
+   * Context geolocation. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  geolocation: Geolocation | undefined;
+
+  /**
+   * Specifies if viewport supports touch events. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  hasTouch: boolean | undefined;
+
+  /**
+   * Credentials for [HTTP authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication).
+   * @see BrowserContextOptions
+   */
+  httpCredentials: HTTPCredentials | undefined;
+
+  /**
+   * Whether to ignore HTTPS errors during navigation. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  ignoreHTTPSErrors: boolean | undefined;
+
+  /**
+   * Whether the `meta viewport` tag is taken into account and touch events are enabled. Not supported in Firefox.
+   * @see BrowserContextOptions
+   */
+  isMobile: boolean | undefined;
+
+  /**
+   * Whether or not to enable JavaScript in the context. Defaults to `true`.
+   * @see BrowserContextOptions
+   */
+  javaScriptEnabled: boolean | undefined;
+
+  /**
+   * User locale, for example `en-GB`, `de-DE`, etc. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  locale: string | undefined;
+
+  /**
+   * Whether to emulate network being offline.
+   * @see BrowserContextOptions
+   */
+  offline: boolean | undefined;
+
+  /**
+   * A list of permissions to grant to all pages in this context. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  permissions: string[] | undefined;
+
+  /**
+   * Proxy setting used for all pages in the test. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  proxy: Proxy | undefined;
+
+  /**
+   * Populates context with given storage state. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  storageState: StorageState | undefined;
+
+  /**
+   * Changes the timezone of the context. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  timezoneId: string | undefined;
+
+  /**
+   * Specific user agent to use in this context.
+   * @see BrowserContextOptions
+   */
+  userAgent: string | undefined;
+
+  /**
+   * Viewport used for all pages in the test. Takes priority over `contextOptions`.
+   * @see BrowserContextOptions
+   */
+  viewport: ViewportSize | undefined;
+
+  /**
+   * Options used to create the context. Other options above (e.g. `viewport`) take priority.
+   * @see BrowserContextOptions
+   */
+  contextOptions: BrowserContextOptions;
+};
+
+
+/**
+ * Arguments available to the test function and all hooks (beforeEach, afterEach, beforeAll, afterAll).
+ */
+export type PlaywrightWorkerArgs = {
+  /**
+   * The Playwright instance.
+   */
+  playwright: typeof import('..');
+
+  /**
+   * Browser instance, shared between multiple tests.
+   */
+  browser: Browser;
+};
+
+/**
+ * Arguments available to the test function and beforeEach/afterEach hooks.
+ */
+export type PlaywrightTestArgs = {
+  /**
+   * BrowserContext instance, created fresh for each test.
+   */
+  context: BrowserContext;
+
+  /**
+   * Page instance, created fresh for each test.
+   */
+  page: Page;
+};
+
+export type PlaywrightTestProject<TestArgs = {}, WorkerArgs = {}> = Project<PlaywrightTestOptions & TestArgs, PlaywrightWorkerOptions & WorkerArgs>;
+export type PlaywrightTestConfig<TestArgs = {}, WorkerArgs = {}> = Config<PlaywrightTestOptions & TestArgs, PlaywrightWorkerOptions & WorkerArgs>;
+
+export * from 'folio';
+
+import type { TestType } from 'folio';
+
+/**
+ * These tests are executed in Playwright environment that launches the browser
+ * and provides a fresh page to each test.
+ */
+export const test: TestType<PlaywrightTestArgs & PlaywrightTestOptions, PlaywrightWorkerArgs & PlaywrightWorkerOptions>;
+export default test;

utils/publish_all_packages.sh

@@ -87,11 +87,13 @@ PLAYWRIGHT_CORE_TGZ="$PWD/playwright-core.tgz"
 PLAYWRIGHT_WEBKIT_TGZ="$PWD/playwright-webkit.tgz"
 PLAYWRIGHT_FIREFOX_TGZ="$PWD/playwright-firefox.tgz"
 PLAYWRIGHT_CHROMIUM_TGZ="$PWD/playwright-chromium.tgz"
+PLAYWRIGHT_TEST_TGZ="$PWD/playwright-test.tgz"
 node ./packages/build_package.js playwright "${PLAYWRIGHT_TGZ}"
 node ./packages/build_package.js playwright-core "${PLAYWRIGHT_CORE_TGZ}"
 node ./packages/build_package.js playwright-webkit "${PLAYWRIGHT_WEBKIT_TGZ}"
 node ./packages/build_package.js playwright-firefox "${PLAYWRIGHT_FIREFOX_TGZ}"
 node ./packages/build_package.js playwright-chromium "${PLAYWRIGHT_CHROMIUM_TGZ}"
+node ./packages/build_package.js playwright-test "${PLAYWRIGHT_TEST_TGZ}"
 
 echo "==================== Publishing version ${VERSION} ================"
 
@@ -100,5 +102,6 @@ npm publish ${PLAYWRIGHT_CORE_TGZ}      --tag="${NPM_PUBLISH_TAG}"
 npm publish ${PLAYWRIGHT_WEBKIT_TGZ}    --tag="${NPM_PUBLISH_TAG}"
 npm publish ${PLAYWRIGHT_FIREFOX_TGZ}   --tag="${NPM_PUBLISH_TAG}"
 npm publish ${PLAYWRIGHT_CHROMIUM_TGZ}  --tag="${NPM_PUBLISH_TAG}"
+npm publish ${PLAYWRIGHT_TEST_TGZ}      --tag="${NPM_PUBLISH_TAG}"
 
 echo "Done."