Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore(testrunner): move out of the repo (#3687)

Browse source
This commit is contained in:
Joel Einbinder 2020-08-29 08:27:32 -07:00 committed by GitHub
parent 555a8d0d10
commit 3cc91093a1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
62 changed files with 41 additions and 3592 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/auto_roll.yml vendored
View file

@ -27,7 +27,7 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --retries=3 --timeout=30000 --global-timeout=5400000 --reporter=dot,json"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --retries=3 --timeout=30000 --global-timeout=5400000 --reporter=dot,json"
env:
BROWSER: ${{ matrix.browser }}
DEBUG: "pw:*,-pw:wrapped*,-pw:test*"

12
.github/workflows/tests.yml vendored
View file

@ -37,7 +37,7 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json && npm run coverage"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json && npm run coverage"
env:
BROWSER: ${{ matrix.browser }}
PWRUNNER_JSON_REPORT: "test-results/report.json"
@ -62,7 +62,7 @@ jobs:
- uses: microsoft/playwright-github-action@v1
- run: npm ci
- run: npm run build
- run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json
- run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json
env:
BROWSER: ${{ matrix.browser }}
PWRUNNER_JSON_REPORT: "test-results/report.json"
@ -90,7 +90,7 @@ jobs:
- uses: microsoft/playwright-github-action@v1
- run: npm ci
- run: npm run build
- run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json
- run: npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json
shell: bash
env:
BROWSER: ${{ matrix.browser }}
@ -141,7 +141,7 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json"
if: ${{ always() }}
env:
BROWSER: ${{ matrix.browser }}
@ -174,7 +174,7 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json"
env:
BROWSER: ${{ matrix.browser }}
PWWIRE: true
@ -206,7 +206,7 @@ jobs:
# XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
# Wrap `npm run` in a subshell to redirect STDERR to file.
# Enable core dumps in the subshell.
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json"
- run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx test-runner test/ --jobs=1 --forbid-only --timeout=30000 --global-timeout=5400000 --retries=3 --reporter=dot,json"
env:
BROWSER: ${{ matrix.browser }}
TRACING: true

1
.gitignore vendored
View file

@ -1,6 +1,5 @@
/node_modules/
/test-results/
/test-runner/test/test-results/
/test/coverage-report
/test/test-user-data-dir*
.local-browsers/

14
package-lock.json generated
View file

@ -1172,6 +1172,20 @@
"fastq": "^1.6.0"
}
},
"@playwright/test-runner": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/@playwright/test-runner/-/test-runner-0.2.1.tgz",
"integrity": "sha512-ZZILhZZSo4Gv2T70HCkHRIUao3vHZpdSjVbjrXUWGaulxzoF77i0omRth7Kie+qBWhOIlpZe1rgBDQW801GTkw==",
"dev": true,
"requires": {
"colors": "^1.4.0",
"expect": "^26.4.2",
"jpeg-js": "^0.4.2",
"pixelmatch": "^5.2.1",
"pngjs": "^5.0.0",
"text-diff": "^1.0.1"
}
},
"@sindresorhus/is": {
"version": "0.14.0",
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz",

17
package.json
View file

@ -9,19 +9,19 @@
"node": ">=10.15.0"
},
"scripts": {
"ctest": "cross-env BROWSER=chromium node test-runner/cli test/",
"ftest": "cross-env BROWSER=firefox node test-runner/cli test/",
"wtest": "cross-env BROWSER=webkit node test-runner/cli test/",
"test": "node test-runner/cli test/",
"ctest": "cross-env BROWSER=chromium test-runner test/",
"ftest": "cross-env BROWSER=firefox test-runner test/",
"wtest": "cross-env BROWSER=webkit test-runner test/",
"test": "test-runner test/",
"eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .",
"tsc": "tsc -p .",
"tsc-installer": "tsc -p ./src/install/tsconfig.json",
"doc": "node utils/doclint/cli.js",
"test-infra": "node test-runner/cli utils/doclint/check_public_api/test/test.js && node test-runner/cli utils/doclint/preprocessor/test.js && npm run test-testrunner",
"test-infra": "test-runner utils/doclint/check_public_api/test/test.js && test-runner utils/doclint/preprocessor/test.js",
"lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra",
"clean": "rimraf lib && rimraf types",
"prepare": "node install-from-github.js",
"build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run build-testrunner && npm run generate-types",
"build": "node utils/runWebpack.js --mode='development' && tsc -p . && npm run generate-types",
"watch": "node utils/watch.js",
"test-types": "npm run generate-types && npx -p typescript@3.7.5 tsc -p utils/generate_types/test/tsconfig.json && npm run typecheck-tests",
"generate-types": "node utils/generate_types/",
@ -30,9 +30,7 @@
"roll-browser": "node utils/roll_browser.js",
"coverage": "node test/checkCoverage.js",
"check-deps": "node utils/check_deps.js",
"show-trace": "node utils/showTestTraces.js",
"build-testrunner": "tsc -p test-runner",
"test-testrunner": "node test-runner/cli test-runner/test"
"show-trace": "node utils/showTestTraces.js"
},
"author": {
"name": "Microsoft Corporation"
@ -55,6 +53,7 @@
"@babel/core": "^7.11.4",
"@babel/preset-env": "^7.11.0",
"@babel/preset-typescript": "^7.10.4",
"@playwright/test-runner": "^0.2.1",
"@types/babel__core": "^7.1.9",
"@types/debug": "^4.1.5",
"@types/extract-zip": "^1.6.2",

2
test-runner/cli.js
View file

@ -1,2 +0,0 @@
#!/usr/bin/env node
module.exports = require('./lib/cli')

8
test-runner/package.json
View file

@ -1,8 +0,0 @@
{
"name": "test-runner",
"version": "0.0.7",
"bin": {
"test-runner": "./cli.js"
},
"main": "./lib/index.js"
}

68
test-runner/src/builtin.fixtures.ts
View file

@ -1,68 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import path from 'path';
import { promisify } from 'util';
import fs from 'fs';
import rimraf from 'rimraf';
import { registerFixture } from './fixtures';
import { Test, Suite } from './test';
interface DescribeFunction {
describe(name: string, inner: () => void): void;
describe(name: string, modifier: (suite: Suite) => any, inner: () => void): void;
}
interface ItFunction<STATE> {
it(name: string, inner: (state: STATE) => Promise<void> | void): void;
it(name: string, modifier: (test: Test) => any, inner: (state: STATE) => Promise<void> | void): void;
}
declare global {
const describe: DescribeFunction['describe'];
const fdescribe: DescribeFunction['describe'];
const xdescribe: DescribeFunction['describe'];
const it: ItFunction<TestState & WorkerState & FixtureParameters>['it'];
const fit: ItFunction<TestState & WorkerState & FixtureParameters>['it'];
const xit: ItFunction<TestState & WorkerState & FixtureParameters>['it'];
const beforeEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise<void>) => void;
const afterEach: (inner: (state: TestState & WorkerState & FixtureParameters) => Promise<void>) => void;
const beforeAll: (inner: (state: WorkerState & FixtureParameters) => Promise<void>) => void;
const afterAll: (inner: (state: WorkerState & FixtureParameters) => Promise<void>) => void;
}
const mkdtempAsync = promisify(fs.mkdtemp);
const removeFolderAsync = promisify(rimraf);
declare global {
interface FixtureParameters {
parallelIndex: number;
}
interface TestState {
tmpDir: string;
}
}
export {parameters, registerFixture, registerWorkerFixture, registerParameter} from './fixtures';
registerFixture('tmpDir', async ({}, test) => {
const tmpDir = await mkdtempAsync(path.join(os.tmpdir(), 'playwright-test-'));
await test(tmpDir);
await removeFolderAsync(tmpDir).catch(e => {});
});

129
test-runner/src/cli.ts
View file

@ -1,129 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import program from 'commander';
import * as fs from 'fs';
import * as path from 'path';
import { run, RunnerConfig } from '.';
import PytestReporter from './reporters/pytest';
import DotReporter from './reporters/dot';
import ListReporter from './reporters/list';
import JSONReporter from './reporters/json';
import { Reporter } from './reporter';
import { Multiplexer } from './reporters/multiplexer';
export const reporters = {
'dot': DotReporter,
'list': ListReporter,
'json': JSONReporter,
'pytest': PytestReporter,
};
program
.version('Version ' + /** @type {any} */ (require)('../package.json').version)
.option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
.option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
.option('--global-timeout <timeout>', 'Specify maximum time this test suite can run (in milliseconds), default: 0 for unlimited', '0')
.option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', String(Math.ceil(require('os').cpus().length / 2)))
.option('--reporter <reporter>', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot')
.option('--repeat-each <repeat-each>', 'Specify how many times to run the tests', '1')
.option('--retries <retries>', 'Specify retry count', '0')
.option('--trial-run', 'Only collect the matching tests and report them as passing')
.option('--quiet', 'Suppress stdio', false)
.option('--debug', 'Run tests in-process for debugging', false)
.option('--output <outputDir>', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results'))
.option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
.option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
.action(async command => {
const testDir = path.resolve(process.cwd(), command.args[0]);
const config: RunnerConfig = {
debug: command.debug,
forbidOnly: command.forbidOnly,
quiet: command.quiet,
grep: command.grep,
jobs: parseInt(command.jobs, 10),
outputDir: command.output,
repeatEach: parseInt(command.repeatEach, 10),
retries: parseInt(command.retries, 10),
snapshotDir: path.join(testDir, '__snapshots__'),
testDir,
timeout: parseInt(command.timeout, 10),
globalTimeout: parseInt(command.globalTimeout, 10),
trialRun: command.trialRun,
updateSnapshots: command.updateSnapshots
};
const reporterList = command.reporter.split(',');
const reporterObjects: Reporter[] = reporterList.map(c => {
if (reporters[c])
return new reporters[c]();
try {
const p = path.resolve(process.cwd(), c);
return new (require(p).default)();
} catch (e) {
console.error('Invalid reporter ' + c, e);
process.exit(1);
}
});
const files = collectFiles(testDir, '', command.args.slice(1));
const result = await run(config, files, new Multiplexer(reporterObjects));
if (result === 'forbid-only') {
console.error('=====================================');
console.error(' --forbid-only found a focused test.');
console.error('=====================================');
process.exit(1);
}
if (result === 'no-tests') {
console.error('=================');
console.error(' no tests found.');
console.error('=================');
process.exit(1);
}
process.exit(result === 'failed' ? 1 : 0);
});
program.parse(process.argv);
function collectFiles(testDir: string, dir: string, filters: string[]): string[] {
const fullDir = path.join(testDir, dir);
if (fs.statSync(fullDir).isFile())
return [fullDir];
const files = [];
for (const name of fs.readdirSync(fullDir)) {
if (fs.lstatSync(path.join(fullDir, name)).isDirectory()) {
files.push(...collectFiles(testDir, path.join(dir, name), filters));
continue;
}
if (!name.endsWith('spec.ts'))
continue;
const relativeName = path.join(dir, name);
const fullName = path.join(testDir, relativeName);
if (!filters.length) {
files.push(fullName);
continue;
}
for (const filter of filters) {
if (relativeName.includes(filter)) {
files.push(fullName);
break;
}
}
}
return files;
}

44
test-runner/src/expect.ts
View file

@ -1,44 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { compare } from './golden';
import { RunnerConfig } from './runnerConfig';
declare global {
const expect: typeof import('expect');
}
declare module 'expect/build/types' {
interface Matchers<R> {
toMatchImage(path: string, options?: { threshold?: number }): R;
}
}
global['expect'] = require('expect');
let testFile: string;
export function initializeImageMatcher(config: RunnerConfig) {
function toMatchImage(received: Buffer, name: string, options?: { threshold?: number }) {
const { pass, message } = compare(received, name, config, testFile, options);
return { pass, message: () => message };
}
expect.extend({ toMatchImage });
}
export function setCurrentTestFile(file: string) {
testFile = file;
}

276
test-runner/src/fixtures.ts
View file

@ -1,276 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import debug from 'debug';
import { RunnerConfig } from './runnerConfig';
import { serializeError, Test, TestResult } from './test';
import { raceAgainstTimeout } from './util';
type Scope = 'test' | 'worker';
type FixtureRegistration = {
name: string;
scope: Scope;
fn: Function;
};
export type TestInfo = {
config: RunnerConfig;
test: Test;
result: TestResult;
};
const registrations = new Map<string, FixtureRegistration>();
const registrationsByFile = new Map<string, FixtureRegistration[]>();
export let parameters: any = {};
export const parameterRegistrations = new Map();
export function setParameters(params: any) {
parameters = Object.assign(parameters, params);
for (const name of Object.keys(params))
registerWorkerFixture(name, async ({}, test) => await test(parameters[name]));
}
class Fixture {
pool: FixturePool;
name: string;
scope: Scope;
fn: Function;
deps: string[];
usages: Set<string>;
hasGeneratorValue: boolean;
value: any;
_teardownFenceCallback: (value?: unknown) => void;
_tearDownComplete: Promise<void>;
_setup = false;
_teardown = false;
constructor(pool: FixturePool, name: string, scope: Scope, fn: any) {
this.pool = pool;
this.name = name;
this.scope = scope;
this.fn = fn;
this.deps = fixtureParameterNames(this.fn);
this.usages = new Set();
this.hasGeneratorValue = name in parameters;
this.value = this.hasGeneratorValue ? parameters[name] : null;
}
async setup(config: RunnerConfig, info?: TestInfo) {
if (this.hasGeneratorValue)
return;
for (const name of this.deps) {
await this.pool.setupFixture(name, config, info);
this.pool.instances.get(name).usages.add(this.name);
}
const params = {};
for (const n of this.deps)
params[n] = this.pool.instances.get(n).value;
let setupFenceFulfill: { (): void; (value?: unknown): void; };
let setupFenceReject: { (arg0: any): any; (reason?: any): void; };
const setupFence = new Promise((f, r) => { setupFenceFulfill = f; setupFenceReject = r; });
const teardownFence = new Promise(f => this._teardownFenceCallback = f);
debug('pw:test:hook')(`setup "${this.name}"`);
const param = info || config;
this._tearDownComplete = this.fn(params, async (value: any) => {
this.value = value;
setupFenceFulfill();
return await teardownFence;
}, param).catch((e: any) => setupFenceReject(e));
await setupFence;
this._setup = true;
}
async teardown() {
if (this.hasGeneratorValue)
return;
if (this._teardown)
return;
this._teardown = true;
for (const name of this.usages) {
const fixture = this.pool.instances.get(name);
if (!fixture)
continue;
await fixture.teardown();
}
if (this._setup) {
debug('pw:test:hook')(`teardown "${this.name}"`);
this._teardownFenceCallback();
await this._tearDownComplete;
}
this.pool.instances.delete(this.name);
}
}
export class FixturePool {
instances: Map<string, Fixture>;
constructor() {
this.instances = new Map();
}
async setupFixture(name: string, config: RunnerConfig, info?: TestInfo) {
let fixture = this.instances.get(name);
if (fixture)
return fixture;
if (!registrations.has(name))
throw new Error('Unknown fixture: ' + name);
const { scope, fn } = registrations.get(name);
fixture = new Fixture(this, name, scope, fn);
this.instances.set(name, fixture);
await fixture.setup(config, info);
return fixture;
}
async teardownScope(scope: string) {
for (const [, fixture] of this.instances) {
if (fixture.scope === scope)
await fixture.teardown();
}
}
async resolveParametersAndRun(fn: Function, config: RunnerConfig, info?: TestInfo) {
const names = fixtureParameterNames(fn);
for (const name of names)
await this.setupFixture(name, config, info);
const params = {};
for (const n of names)
params[n] = this.instances.get(n).value;
return fn(params);
}
async runTestWithFixturesAndTimeout(fn: Function, timeout: number, info: TestInfo) {
const { timedOut } = await raceAgainstTimeout(this._runTestWithFixtures(fn, info), timeout);
// Do not overwrite test failure upon timeout in fixture.
if (timedOut && info.result.status === 'passed')
info.result.status = 'timedOut';
}
async _runTestWithFixtures(fn: Function, info: TestInfo) {
try {
await this.resolveParametersAndRun(fn, info.config, info);
info.result.status = 'passed';
} catch (error) {
// Prefer original error to the fixture teardown error or timeout.
if (info.result.status === 'passed') {
info.result.status = 'failed';
info.result.error = serializeError(error);
}
}
try {
await this.teardownScope('test');
} catch (error) {
// Prefer original error to the fixture teardown error or timeout.
if (info.result.status === 'passed') {
info.result.status = 'failed';
info.result.error = serializeError(error);
}
}
}
}
export function fixturesForCallback(callback: Function): string[] {
const names = new Set<string>();
const visit = (callback: Function) => {
for (const name of fixtureParameterNames(callback)) {
if (name in names)
continue;
names.add(name);
if (!registrations.has(name))
throw new Error('Using undefined fixture ' + name);
const { fn } = registrations.get(name);
visit(fn);
}
};
visit(callback);
const result = [...names];
result.sort();
return result;
}
function fixtureParameterNames(fn: Function): string[] {
const text = fn.toString();
const match = text.match(/async(?:\s+function)?\s*\(\s*{\s*([^}]*)\s*}/);
if (!match || !match[1].trim())
return [];
const signature = match[1];
return signature.split(',').map((t: string) => t.trim());
}
function innerRegisterFixture(name: string, scope: Scope, fn: Function, caller: Function) {
const obj = {stack: ''};
Error.captureStackTrace(obj, caller);
const stackFrame = obj.stack.split('\n')[2];
const location = stackFrame.replace(/.*at Object.<anonymous> \((.*)\)/, '$1');
const file = location.replace(/^(.+):\d+:\d+$/, '$1');
const registration = { name, scope, fn, file, location };
registrations.set(name, registration);
if (!registrationsByFile.has(file))
registrationsByFile.set(file, []);
registrationsByFile.get(file).push(registration);
}
export function registerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise<void>, info: TestInfo) => Promise<void>) {
innerRegisterFixture(name, 'test', fn, registerFixture);
}
export function registerWorkerFixture(name: string, fn: (params: any, runTest: (arg: any) => Promise<void>, config: RunnerConfig) => Promise<void>) {
innerRegisterFixture(name, 'worker', fn, registerWorkerFixture);
}
export function registerParameter(name: string, fn: () => any) {
registerWorkerFixture(name, async ({}: any, test: Function) => await test(parameters[name]));
parameterRegistrations.set(name, fn);
}
function collectRequires(file: string, result: Set<string>) {
if (result.has(file))
return;
result.add(file);
const cache = require.cache[file];
if (!cache)
return;
const deps = cache.children.map((m: { id: any; }) => m.id).slice().reverse();
for (const dep of deps)
collectRequires(dep, result);
}
export function lookupRegistrations(file: string, scope: Scope) {
const deps = new Set<string>();
collectRequires(file, deps);
const allDeps = [...deps].reverse();
const result = new Map();
for (const dep of allDeps) {
const registrationList = registrationsByFile.get(dep);
if (!registrationList)
continue;
for (const r of registrationList) {
if (scope && r.scope !== scope)
continue;
result.set(r.name, r);
}
}
return result;
}
export function rerunRegistrations(file: string, scope: Scope) {
// When we are running several tests in the same worker, we should re-run registrations before
// each file. That way we erase potential fixture overrides from the previous test runs.
for (const registration of lookupRegistrations(file, scope).values())
registrations.set(registration.name, registration);
}

152
test-runner/src/golden.ts
View file

@ -1,152 +0,0 @@
/**
* Copyright 2017 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import c from 'colors/safe';
import fs from 'fs';
import jpeg from 'jpeg-js';
import path from 'path';
import pixelmatch from 'pixelmatch';
import { PNG } from 'pngjs';
import Diff from 'text-diff';
import { RunnerConfig } from './runnerConfig';
const extensionToMimeType = {
'png': 'image/png',
'txt': 'text/plain',
'jpg': 'image/jpeg',
'jpeg': 'image/jpeg',
};
const GoldenComparators = {
'image/png': compareImages,
'image/jpeg': compareImages,
'text/plain': compareText
};
function compareImages(actualBuffer: Buffer, expectedBuffer: Buffer, mimeType: string, options = {}): { diff?: object; errorMessage?: string; } | null {
if (!actualBuffer || !(actualBuffer instanceof Buffer))
return { errorMessage: 'Actual result should be Buffer.' };
const actual = mimeType === 'image/png' ? PNG.sync.read(actualBuffer) : jpeg.decode(actualBuffer);
const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer);
if (expected.width !== actual.width || expected.height !== actual.height) {
return {
errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. `
};
}
const diff = new PNG({width: expected.width, height: expected.height});
const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { threshold: 0.2, ...options });
return count > 0 ? { diff: PNG.sync.write(diff) } : null;
}
function compareText(actual: Buffer, expectedBuffer: Buffer): { diff?: object; errorMessage?: string; diffExtension?: string; } | null {
if (typeof actual !== 'string')
return { errorMessage: 'Actual result should be string' };
const expected = expectedBuffer.toString('utf-8');
if (expected === actual)
return null;
const diff = new Diff();
const result = diff.main(expected, actual);
diff.cleanupSemantic(result);
let html = diff.prettyHtml(result);
const diffStylePath = path.join(__dirname, 'diffstyle.css');
html = `<link rel="stylesheet" href="file://${diffStylePath}">` + html;
return {
diff: html,
diffExtension: '.html'
};
}
export function compare(actual: Buffer, name: string, config: RunnerConfig, testFile: string, options?: { threshold?: number }): { pass: boolean; message?: string; } {
let expectedPath: string;
const relativeTestFile = path.relative(config.testDir, testFile);
const testAssetsDir = relativeTestFile.replace(/\.spec\.[jt]s/, '');
if (path.isAbsolute(name))
expectedPath = name;
else
expectedPath = path.join(config.snapshotDir, testAssetsDir, name);
if (!fs.existsSync(expectedPath)) {
fs.mkdirSync(path.dirname(expectedPath), { recursive: true });
fs.writeFileSync(expectedPath, actual);
return {
pass: false,
message: expectedPath + ' is missing in golden results, writing actual.'
};
}
const expected = fs.readFileSync(expectedPath);
const extension = path.extname(expectedPath).substring(1);
const mimeType = extensionToMimeType[extension];
const comparator = GoldenComparators[mimeType];
if (!comparator) {
return {
pass: false,
message: 'Failed to find comparator with type ' + mimeType + ': ' + expectedPath,
};
}
const result = comparator(actual, expected, mimeType, options);
if (!result)
return { pass: true };
if (config.updateSnapshots) {
fs.mkdirSync(path.dirname(expectedPath), { recursive: true });
fs.writeFileSync(expectedPath, actual);
return {
pass: true,
message: expectedPath + ' running with --update-snapshots, writing actual.'
};
}
let actualPath;
let diffPath;
if (path.isAbsolute(name)) {
actualPath = addSuffix(expectedPath, '-actual');
diffPath = addSuffix(expectedPath, '-diff', result.diffExtension);
} else {
const outputPath = path.join(config.outputDir, testAssetsDir, name);
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
const expectedPathOut = addSuffix(outputPath, '-expected');
actualPath = addSuffix(outputPath, '-actual');
diffPath = addSuffix(outputPath, '-diff', result.diffExtension);
fs.writeFileSync(expectedPathOut, expected);
}
fs.writeFileSync(actualPath, actual);
if (result.diff)
fs.writeFileSync(diffPath, result.diff);
const output = [
c.red(`Image comparison failed:`),
];
if (result.errorMessage)
output.push(' ' + result.errorMessage);
output.push('');
output.push(`Expected: ${c.yellow(expectedPath)}`);
output.push(`Received: ${c.yellow(actualPath)}`);
if (result.diff)
output.push(` Diff: ${c.yellow(diffPath)}`);
return {
pass: false,
message: output.join('\n'),
};
}
function addSuffix(filePath: string, suffix: string, customExtension?: string): string {
const dirname = path.dirname(filePath);
const ext = path.extname(filePath);
const name = path.basename(filePath, ext);
return path.join(dirname, name + suffix + (customExtension || ext));
}

120
test-runner/src/index.ts
View file

@ -1,120 +0,0 @@
/**
* Copyright 2019 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as fs from 'fs';
import * as path from 'path';
import rimraf from 'rimraf';
import { promisify } from 'util';
import './builtin.fixtures';
import './expect';
import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT, TestInfo } from './fixtures';
import { Reporter } from './reporter';
import { Runner } from './runner';
import { RunnerConfig } from './runnerConfig';
import { Suite } from './test';
import { Matrix, TestCollector } from './testCollector';
import { installTransform } from './transform';
import { raceAgainstTimeout } from './util';
export { parameters, registerParameter } from './fixtures';
export { Reporter } from './reporter';
export { RunnerConfig } from './runnerConfig';
export { Suite, Test } from './test';
const removeFolderAsync = promisify(rimraf);
declare global {
interface WorkerState {
}
interface TestState {
}
interface FixtureParameters {
}
}
const beforeFunctions: Function[] = [];
const afterFunctions: Function[] = [];
let matrix: Matrix = {};
global['before'] = (fn: Function) => beforeFunctions.push(fn);
global['after'] = (fn: Function) => afterFunctions.push(fn);
global['matrix'] = (m: Matrix) => matrix = m;
export function registerFixture<T extends keyof TestState>(name: T, fn: (params: FixtureParameters & WorkerState & TestState, runTest: (arg: TestState[T]) => Promise<void>, info: TestInfo) => Promise<void>) {
registerFixtureT(name, fn);
}
export function registerWorkerFixture<T extends keyof(WorkerState & FixtureParameters)>(name: T, fn: (params: FixtureParameters & WorkerState, runTest: (arg: (WorkerState & FixtureParameters)[T]) => Promise<void>, config: RunnerConfig) => Promise<void>) {
registerWorkerFixtureT(name, fn);
}
type RunResult = 'passed' | 'failed' | 'forbid-only' | 'no-tests';
export async function run(config: RunnerConfig, files: string[], reporter: Reporter): Promise<RunResult> {
if (!config.trialRun) {
await removeFolderAsync(config.outputDir).catch(e => {});
fs.mkdirSync(config.outputDir, { recursive: true });
}
const revertBabelRequire = installTransform();
let hasSetup = false;
try {
hasSetup = fs.statSync(path.join(config.testDir, 'setup.js')).isFile();
} catch (e) {
}
try {
hasSetup = hasSetup || fs.statSync(path.join(config.testDir, 'setup.ts')).isFile();
} catch (e) {
}
if (hasSetup)
require(path.join(config.testDir, 'setup'));
revertBabelRequire();
const testCollector = new TestCollector(files, matrix, config);
const suite = testCollector.suite;
if (config.forbidOnly) {
const hasOnly = suite.findTest(t => t._only) || suite.eachSuite(s => s._only);
if (hasOnly)
return 'forbid-only';
}
const total = suite.total();
if (!total)
return 'no-tests';
const { result, timedOut } = await raceAgainstTimeout(runTests(config, suite, reporter), config.globalTimeout);
if (timedOut) {
reporter.onTimeout(config.globalTimeout);
process.exit(1);
}
return result;
}
async function runTests(config: RunnerConfig, suite: Suite, reporter: Reporter) {
// Trial run does not need many workers, use one.
const jobs = (config.trialRun || config.debug) ? 1 : config.jobs;
const runner = new Runner(suite, { ...config, jobs }, reporter);
try {
for (const f of beforeFunctions)
await f();
await runner.run();
await runner.stop();
} finally {
for (const f of afterFunctions)
await f();
}
return suite.findTest(test => !test._ok()) ? 'failed' : 'passed';
}

28
test-runner/src/reporter.ts
View file

@ -1,28 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { RunnerConfig } from './runnerConfig';
import { Suite, Test, TestResult } from './test';
export interface Reporter {
onBegin(config: RunnerConfig, suite: Suite): void;
onTestBegin(test: Test): void;
onTestStdOut(test: Test, chunk: string | Buffer): void;
onTestStdErr(test: Test, chunk: string | Buffer): void;
onTestEnd(test: Test, result: TestResult): void;
onTimeout(timeout: number): void;
onEnd(): void;
}

201
test-runner/src/reporters/base.ts
View file

@ -1,201 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { codeFrameColumns } from '@babel/code-frame';
import colors from 'colors/safe';
import fs from 'fs';
import milliseconds from 'ms';
import os from 'os';
import path from 'path';
import StackUtils from 'stack-utils';
import terminalLink from 'terminal-link';
import { Reporter } from '../reporter';
import { RunnerConfig } from '../runnerConfig';
import { Suite, Test, TestResult } from '../test';
const stackUtils = new StackUtils();
export class BaseReporter implements Reporter {
skipped: Test[] = [];
asExpected: Test[] = [];
unexpected = new Set<Test>();
expectedFlaky: Test[] = [];
unexpectedFlaky: Test[] = [];
duration = 0;
startTime: number;
config: RunnerConfig;
suite: Suite;
timeout: number;
constructor() {
process.on('SIGINT', async () => {
this.epilogue();
process.exit(130);
});
}
onBegin(config: RunnerConfig, suite: Suite) {
this.startTime = Date.now();
this.config = config;
this.suite = suite;
}
onTestBegin(test: Test) {
}
onTestStdOut(test: Test, chunk: string | Buffer) {
if (!this.config.quiet)
process.stdout.write(chunk);
}
onTestStdErr(test: Test, chunk: string | Buffer) {
if (!this.config.quiet)
process.stderr.write(chunk);
}
onTestEnd(test: Test, result: TestResult) {
if (result.status === 'skipped') {
this.skipped.push(test);
return;
}
if (result.status === result.expectedStatus) {
if (test.results.length === 1) {
// as expected from the first attempt
this.asExpected.push(test);
} else {
// as expected after unexpected -> flaky.
if (test.isFlaky())
this.expectedFlaky.push(test);
else
this.unexpectedFlaky.push(test);
}
return;
}
if (result.status === 'passed' || result.status === 'timedOut' || test.results.length === this.config.retries + 1) {
// We made as many retries as we could, still failing.
this.unexpected.add(test);
}
}
onTimeout(timeout: number) {
this.timeout = timeout;
}
onEnd() {
this.duration = Date.now() - this.startTime;
}
epilogue() {
console.log('');
console.log(colors.green(` ${this.asExpected.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
if (this.skipped.length)
console.log(colors.yellow(` ${this.skipped.length} skipped`));
const filteredUnexpected = [...this.unexpected].filter(t => !t._hasResultWithStatus('timedOut'));
if (filteredUnexpected.length) {
console.log(colors.red(` ${filteredUnexpected.length} failed`));
console.log('');
this._printFailures(filteredUnexpected);
}
const allFlaky = this.expectedFlaky.length + this.unexpectedFlaky.length;
if (allFlaky) {
console.log(colors.red(` ${allFlaky} flaky`));
if (this.unexpectedFlaky.length) {
console.log('');
this._printFailures(this.unexpectedFlaky);
}
}
const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut'));
if (timedOut.length) {
console.log(colors.red(` ${timedOut.length} timed out`));
console.log('');
this._printFailures(timedOut);
}
console.log('');
if (this.timeout) {
console.log(colors.red(` Timed out waiting ${this.timeout / 1000}s for the entire test run`));
console.log('');
}
}
private _printFailures(failures: Test[]) {
failures.forEach((test, index) => {
console.log(this.formatFailure(test, index + 1));
});
}
formatFailure(test: Test, index?: number): string {
const tokens: string[] = [];
const relativePath = path.relative(process.cwd(), test.file);
const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : '';
const header = ` ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}${passedUnexpectedlySuffix}`;
tokens.push(colors.bold(colors.red(header)));
for (const result of test.results) {
if (result.status === 'passed')
continue;
if (result.status === 'timedOut') {
tokens.push('');
tokens.push(indent(colors.red(`Timeout of ${test._timeout}ms exceeded.`), ' '));
} else {
const stack = result.error.stack;
if (stack) {
tokens.push('');
const messageLocation = result.error.stack.indexOf(result.error.message);
const preamble = result.error.stack.substring(0, messageLocation + result.error.message.length);
tokens.push(indent(preamble, ' '));
const position = positionInFile(stack, test.file);
if (position) {
const source = fs.readFileSync(test.file, 'utf8');
tokens.push('');
tokens.push(indent(codeFrameColumns(source, {
start: position,
},
{ highlightCode: true}
), ' '));
}
tokens.push('');
tokens.push(indent(colors.dim(stack.substring(preamble.length + 1)), ' '));
} else {
tokens.push('');
tokens.push(indent(String(result.error), ' '));
}
}
break;
}
tokens.push('');
return tokens.join('\n');
}
}
function indent(lines: string, tab: string) {
return lines.replace(/^/gm, tab);
}
function positionInFile(stack: string, file: string): { column: number; line: number; } {
for (const line of stack.split('\n')) {
const parsed = stackUtils.parseLine(line);
if (!parsed)
continue;
if (path.resolve(process.cwd(), parsed.file) === file)
return {column: parsed.column, line: parsed.line};
}
return null;
}

44
test-runner/src/reporters/dot.ts
View file

@ -1,44 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import { BaseReporter } from './base';
import { Test, TestResult } from '../test';
class DotReporter extends BaseReporter {
onTestEnd(test: Test, result: TestResult) {
super.onTestEnd(test, result);
switch (result.status) {
case 'skipped': process.stdout.write(colors.yellow('∘')); break;
case 'passed': process.stdout.write(result.status === result.expectedStatus ? colors.green('·') : colors.red('P')); break;
case 'failed': process.stdout.write(result.status === result.expectedStatus ? colors.green('f') : colors.red('F')); break;
case 'timedOut': process.stdout.write(colors.red('T')); break;
}
}
onTimeout(timeout) {
super.onTimeout(timeout);
this.onEnd();
}
onEnd() {
super.onEnd();
process.stdout.write('\n');
this.epilogue();
}
}
export default DotReporter;

82
test-runner/src/reporters/json.ts
View file

@ -1,82 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { BaseReporter } from './base';
import { Suite, Test, TestResult } from '../test';
import * as fs from 'fs';
class JSONReporter extends BaseReporter {
onTimeout(timeout) {
super.onTimeout(timeout);
this.onEnd();
}
onEnd() {
super.onEnd();
const result = {
config: this.config,
suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s)
};
const report = JSON.stringify(result, undefined, 2);
if (process.env.PWRUNNER_JSON_REPORT)
fs.writeFileSync(process.env.PWRUNNER_JSON_REPORT, report);
else
console.log(report);
}
private _serializeSuite(suite: Suite): any {
if (!suite.findTest(test => true))
return null;
const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s);
return {
title: suite.title,
file: suite.file,
configuration: suite.configuration,
tests: suite.tests.map(test => this._serializeTest(test)),
suites: suites.length ? suites : undefined
};
}
private _serializeTest(test: Test): any {
return {
title: test.title,
file: test.file,
only: test.isOnly(),
slow: test.isSlow(),
timeout: test.timeout(),
results: test.results.map(r => this._serializeTestResult(r))
};
}
private _serializeTestResult(result: TestResult): any {
return {
status: result.status,
duration: result.duration,
error: result.error,
stdout: result.stdout.map(s => stdioEntry(s)),
stderr: result.stderr.map(s => stdioEntry(s)),
data: result.data
};
}
}
function stdioEntry(s: string | Buffer): any {
if (typeof s === 'string')
return { text: s };
return { buffer: s.toString('base64') };
}
export default JSONReporter;

57
test-runner/src/reporters/list.ts
View file

@ -1,57 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import { BaseReporter } from './base';
import { RunnerConfig } from '../runnerConfig';
import { Suite, Test, TestResult } from '../test';
class ListReporter extends BaseReporter {
_failure = 0;
onBegin(config: RunnerConfig, suite: Suite) {
super.onBegin(config, suite);
console.log();
}
onTestBegin(test: Test) {
super.onTestBegin(test);
process.stdout.write(' ' + colors.gray(test.fullTitle() + ': '));
}
onTestEnd(test: Test, result: TestResult) {
super.onTestEnd(test, result);
let text = '';
if (result.status === 'skipped') {
text = colors.green(' - ') + colors.cyan(test.fullTitle());
} else {
const statusMark = result.status === 'passed' ? colors.green(' ✓ ') : colors.red(' x ');
if (result.status === result.expectedStatus)
text = '\u001b[2K\u001b[0G' + statusMark + colors.gray(test.fullTitle());
else
text = '\u001b[2K\u001b[0G' + colors.red(` ${++this._failure}) ` + test.fullTitle());
}
process.stdout.write(text + '\n');
}
onEnd() {
super.onEnd();
process.stdout.write('\n');
this.epilogue();
}
}
export default ListReporter;

62
test-runner/src/reporters/multiplexer.ts
View file

@ -1,62 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { RunnerConfig } from '../runnerConfig';
import { Suite, Test, TestResult } from '../test';
import { Reporter } from '../reporter';
export class Multiplexer implements Reporter {
private _reporters: Reporter[];
constructor(reporters: Reporter[]) {
this._reporters = reporters;
}
onBegin(config: RunnerConfig, suite: Suite) {
for (const reporter of this._reporters)
reporter.onBegin(config, suite);
}
onTestBegin(test: Test) {
for (const reporter of this._reporters)
reporter.onTestBegin(test);
}
onTestStdOut(test: Test, chunk: string | Buffer) {
for (const reporter of this._reporters)
reporter.onTestStdOut(test, chunk);
}
onTestStdErr(test: Test, chunk: string | Buffer) {
for (const reporter of this._reporters)
reporter.onTestStdErr(test, chunk);
}
onTestEnd(test: Test, result: TestResult) {
for (const reporter of this._reporters)
reporter.onTestEnd(test, result);
}
onTimeout(timeout: number) {
for (const reporter of this._reporters)
reporter.onTimeout(timeout);
}
onEnd() {
for (const reporter of this._reporters)
reporter.onEnd();
}
}

254
test-runner/src/reporters/pytest.ts
View file

@ -1,254 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import milliseconds from 'ms';
import * as path from 'path';
import { Test, Suite, Configuration, TestResult } from '../test';
import { BaseReporter } from './base';
import { RunnerConfig } from '../runnerConfig';
const cursorPrevLine = '\u001B[F';
const eraseLine = '\u001B[2K';
type Row = {
id: string;
relativeFile: string;
configuration: string;
ordinal: number;
track: string[];
total: number;
failed: boolean;
startTime: number;
finishTime: number;
};
const statusRows = 2;
class PytestReporter extends BaseReporter {
private _rows = new Map<string, Row>();
private _suiteIds = new Map<Suite, string>();
private _lastOrdinal = 0;
private _visibleRows: number;
private _total: number;
private _progress: string[] = [];
private _throttler = new Throttler(250, () => this._repaint());
onBegin(config: RunnerConfig, rootSuite: Suite) {
super.onBegin(config, rootSuite);
this._total = rootSuite.total();
const jobs = Math.min(config.jobs, rootSuite.suites.length);
this._visibleRows = jobs + Math.min(jobs, 3); // 3 buffer rows for completed (green) workers.
for (let i = 0; i < this._visibleRows + statusRows; ++i) // 4 rows for status
process.stdout.write('\n');
for (const s of rootSuite.suites) {
const relativeFile = path.relative(this.config.testDir, s.file);
const configurationString = serializeConfiguration(s.configuration);
const id = relativeFile + `::[${configurationString}]`;
this._suiteIds.set(s, id);
const row = {
id,
relativeFile,
configuration: configurationString,
ordinal: this._lastOrdinal++,
track: [],
total: s.total(),
failed: false,
startTime: 0,
finishTime: 0,
};
this._rows.set(id, row);
}
}
onTestBegin(test: Test) {
super.onTestBegin(test);
const row = this._rows.get(this._id(test));
if (!row.startTime)
row.startTime = Date.now();
}
onTestStdOut(test: Test, chunk: string | Buffer) {
this._repaint(chunk);
}
onTestStdErr(test: Test, chunk: string | Buffer) {
this._repaint(chunk);
}
onTestEnd(test: Test, result: TestResult) {
super.onTestEnd(test, result);
switch (result.status) {
case 'skipped': {
this._append(test, colors.yellow('∘'));
this._progress.push('S');
this._throttler.schedule();
break;
}
case 'passed': {
this._append(test, colors.green('✓'));
this._progress.push('P');
this._throttler.schedule();
break;
}
case 'failed':
// fall through
case 'timedOut': {
const title = result.status === 'timedOut' ? colors.red('T') : colors.red('F');
const row = this._append(test, title);
row.failed = true;
this._progress.push('F');
this._repaint(this.formatFailure(test) + '\n');
break;
}
}
}
private _append(test: Test, s: string): Row {
const testId = this._id(test);
const row = this._rows.get(testId);
row.track.push(s);
if (row.track.length === row.total)
row.finishTime = Date.now();
return row;
}
private _repaint(prependChunk?: string | Buffer) {
const rowList = [...this._rows.values()];
const running = rowList.filter(r => r.startTime && !r.finishTime);
const finished = rowList.filter(r => r.finishTime).sort((a, b) => b.finishTime - a.finishTime);
const finishedToPrint = finished.slice(0, this._visibleRows - running.length);
const lines = [];
for (const row of finishedToPrint.concat(running)) {
const remaining = row.total - row.track.length;
const remainder = '·'.repeat(remaining);
let title = row.relativeFile;
if (row.finishTime) {
if (row.failed)
title = colors.red(row.relativeFile);
else
title = colors.green(row.relativeFile);
}
const configuration = ` [${colors.gray(row.configuration)}]`;
lines.push(' ' + title + configuration + ' ' + row.track.join('') + colors.gray(remainder));
}
const status = [];
if (this.asExpected.length)
status.push(colors.green(`${this.asExpected.length} as expected`));
if (this.skipped.length)
status.push(colors.yellow(`${this.skipped.length} skipped`));
const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut'));
if (this.unexpected.size - timedOut.length)
status.push(colors.red(`${this.unexpected.size - timedOut.length} unexpected failures`));
if (timedOut.length)
status.push(colors.red(`${timedOut.length} timed out`));
status.push(colors.dim(`(${milliseconds(Date.now() - this.startTime)})`));
for (let i = lines.length; i < this._visibleRows; ++i)
lines.push('');
lines.push(this._paintProgress(this._progress.length, this._total));
lines.push(status.join(' '));
lines.push('');
process.stdout.write((cursorPrevLine + eraseLine).repeat(this._visibleRows + statusRows));
if (prependChunk)
process.stdout.write(prependChunk);
process.stdout.write(lines.join('\n'));
}
private _id(test: Test): string {
for (let suite = test.parent; suite; suite = suite.parent) {
if (this._suiteIds.has(suite))
return this._suiteIds.get(suite);
}
return '';
}
private _paintProgress(worked: number, total: number) {
const length = Math.min(total, 80);
const cellSize = Math.ceil(total / length);
const cellNum = (total / cellSize) | 0;
const bars: string[] = [];
for (let i = 0; i < cellNum; ++i) {
let bar = blankBar;
if (worked < cellSize * i) {
bars.push(bar);
continue;
}
bar = greenBar;
for (let j = i * cellSize; j < worked && j < (i + 1) * cellSize; ++j) {
if (worked < j)
continue;
if (this._progress[j] === 'F') {
bar = redBar;
break;
}
if (this._progress[j] === 'S') {
bar = yellowBar;
break;
}
}
bars.push(bar);
}
return '[' + bars.join('') + '] ' + worked + '/' + total;
}
}
const blankBar = '-';
const redBar = colors.red('▇');
const greenBar = colors.green('▇');
const yellowBar = colors.yellow('▇');
function serializeConfiguration(configuration: Configuration): string {
const tokens = [];
for (const { name, value } of configuration)
tokens.push(`${name}=${value}`);
return tokens.join(', ');
}
class Throttler {
private _timeout: number;
private _callback: () => void;
private _lastFire = 0;
private _timer: NodeJS.Timeout | null = null;
constructor(timeout: number, callback: () => void) {
this._timeout = timeout;
this._callback = callback;
}
schedule() {
const time = Date.now();
const timeRemaining = this._lastFire + this._timeout - time;
if (timeRemaining <= 0) {
this._fire();
return;
}
if (!this._timer)
this._timer = setTimeout(() => this._fire(), timeRemaining);
}
private _fire() {
this._timer = null;
this._lastFire = Date.now();
this._callback();
}
}
export default PytestReporter;

327
test-runner/src/runner.ts
View file

@ -1,327 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import child_process from 'child_process';
import crypto from 'crypto';
import path from 'path';
import { EventEmitter } from 'events';
import { lookupRegistrations, FixturePool } from './fixtures';
import { Suite, Test, TestResult } from './test';
import { TestRunnerEntry } from './testRunner';
import { RunnerConfig } from './runnerConfig';
import { Reporter } from './reporter';
export class Runner {
private _workers = new Set<Worker>();
private _freeWorkers: Worker[] = [];
private _workerClaimers: (() => void)[] = [];
private _testById = new Map<string, { test: Test, result: TestResult }>();
private _queue: TestRunnerEntry[] = [];
private _stopCallback: () => void;
readonly _config: RunnerConfig;
private _suite: Suite;
private _reporter: Reporter;
constructor(suite: Suite, config: RunnerConfig, reporter: Reporter) {
this._config = config;
this._reporter = reporter;
this._suite = suite;
for (const suite of this._suite.suites) {
suite.findTest(test => {
this._testById.set(test._id, { test, result: test._appendResult() });
});
}
if (process.stdout.isTTY) {
const total = suite.total();
console.log();
const jobs = Math.min(config.jobs, suite.suites.length);
console.log(`Running ${total} test${total > 1 ? 's' : ''} using ${jobs} worker${jobs > 1 ? 's' : ''}`);
}
}
_filesSortedByWorkerHash(): TestRunnerEntry[] {
const result: TestRunnerEntry[] = [];
for (const suite of this._suite.suites) {
const ids: string[] = [];
suite.findTest(test => ids.push(test._id) && false);
if (!ids.length)
continue;
result.push({
ids,
file: suite.file,
configuration: suite.configuration,
configurationString: suite._configurationString,
hash: suite._configurationString + '@' + computeWorkerHash(suite.file)
});
}
result.sort((a, b) => a.hash < b.hash ? -1 : (a.hash === b.hash ? 0 : 1));
return result;
}
async run() {
this._reporter.onBegin(this._config, this._suite);
this._queue = this._filesSortedByWorkerHash();
// Loop in case job schedules more jobs
while (this._queue.length)
await this._dispatchQueue();
this._reporter.onEnd();
}
async _dispatchQueue() {
const jobs = [];
while (this._queue.length) {
const entry = this._queue.shift();
const requiredHash = entry.hash;
let worker = await this._obtainWorker();
while (worker.hash && worker.hash !== requiredHash) {
this._restartWorker(worker);
worker = await this._obtainWorker();
}
jobs.push(this._runJob(worker, entry));
}
await Promise.all(jobs);
}
async _runJob(worker: Worker, entry: TestRunnerEntry) {
worker.run(entry);
let doneCallback;
const result = new Promise(f => doneCallback = f);
worker.once('done', params => {
// We won't file remaining if:
// - there are no remaining
// - we are here not because something failed
// - no unrecoverable worker error
if (!params.remaining.length && !params.failedTestId && !params.fatalError) {
this._workerAvailable(worker);
doneCallback();
return;
}
// When worker encounters error, we will restart it.
this._restartWorker(worker);
// In case of fatal error, we are done with the entry.
if (params.fatalError) {
// Report all the tests are failing with this error.
for (const id of entry.ids) {
const { test, result } = this._testById.get(id);
this._reporter.onTestBegin(test);
result.status = 'failed';
result.error = params.fatalError;
this._reporter.onTestEnd(test, result);
}
doneCallback();
return;
}
const remaining = params.remaining;
// Only retry expected failures, not passes and only if the test failed.
if (this._config.retries && params.failedTestId) {
const pair = this._testById.get(params.failedTestId);
if (pair.result.expectedStatus === 'passed' && pair.test.results.length < this._config.retries + 1) {
pair.result = pair.test._appendResult();
remaining.unshift(pair.test._id);
}
}
if (remaining.length)
this._queue.unshift({ ...entry, ids: remaining });
// This job is over, we just scheduled another one.
doneCallback();
});
return result;
}
async _obtainWorker() {
// If there is worker, use it.
if (this._freeWorkers.length)
return this._freeWorkers.pop();
// If we can create worker, create it.
if (this._workers.size < this._config.jobs)
this._createWorker();
// Wait for the next available worker.
await new Promise(f => this._workerClaimers.push(f));
return this._freeWorkers.pop();
}
async _workerAvailable(worker) {
this._freeWorkers.push(worker);
if (this._workerClaimers.length) {
const callback = this._workerClaimers.shift();
callback();
}
}
_createWorker() {
const worker = this._config.debug ? new InProcessWorker(this) : new OopWorker(this);
worker.on('testBegin', params => {
const { test } = this._testById.get(params.id);
test._skipped = params.skipped;
test._flaky = params.flaky;
test._slow = params.slow;
test._expectedStatus = params.expectedStatus;
this._reporter.onTestBegin(test);
});
worker.on('testEnd', params => {
const workerResult: TestResult = params.result;
// We were accumulating these below.
delete workerResult.stdout;
delete workerResult.stderr;
const { test, result } = this._testById.get(params.id);
Object.assign(result, workerResult);
this._reporter.onTestEnd(test, result);
});
worker.on('testStdOut', params => {
const chunk = chunkFromParams(params);
const { test, result } = this._testById.get(params.id);
result.stdout.push(chunk);
this._reporter.onTestStdOut(test, chunk);
});
worker.on('testStdErr', params => {
const chunk = chunkFromParams(params);
const { test, result } = this._testById.get(params.id);
result.stderr.push(chunk);
this._reporter.onTestStdErr(test, chunk);
});
worker.on('exit', () => {
this._workers.delete(worker);
if (this._stopCallback && !this._workers.size)
this._stopCallback();
});
this._workers.add(worker);
worker.init().then(() => this._workerAvailable(worker));
}
async _restartWorker(worker) {
await worker.stop();
this._createWorker();
}
async stop() {
const result = new Promise(f => this._stopCallback = f);
for (const worker of this._workers)
worker.stop();
await result;
}
}
let lastWorkerId = 0;
class Worker extends EventEmitter {
runner: Runner;
hash: string;
constructor(runner) {
super();
this.runner = runner;
}
run(entry: TestRunnerEntry) {
}
stop() {
}
}
class OopWorker extends Worker {
process: child_process.ChildProcess;
stdout: any[];
stderr: any[];
constructor(runner: Runner) {
super(runner);
this.process = child_process.fork(path.join(__dirname, 'worker.js'), {
detached: false,
env: {
FORCE_COLOR: process.stdout.isTTY ? '1' : '0',
DEBUG_COLORS: process.stdout.isTTY ? '1' : '0',
...process.env
},
// Can't pipe since piping slows down termination for some reason.
stdio: ['ignore', 'ignore', 'ignore', 'ipc']
});
this.process.on('exit', () => this.emit('exit'));
this.process.on('error', e => {}); // do not yell at a send to dead process.
this.process.on('message', message => {
const { method, params } = message;
this.emit(method, params);
});
}
async init() {
this.process.send({ method: 'init', params: { workerId: lastWorkerId++, ...this.runner._config } });
await new Promise(f => this.process.once('message', f)); // Ready ack
}
run(entry: TestRunnerEntry) {
this.hash = entry.hash;
this.process.send({ method: 'run', params: { entry, config: this.runner._config } });
}
stop() {
this.process.send({ method: 'stop' });
}
}
class InProcessWorker extends Worker {
fixturePool: FixturePool;
constructor(runner: Runner) {
super(runner);
this.fixturePool = require('./testRunner').fixturePool as FixturePool;
}
async init() {
const { initializeImageMatcher } = require('./expect');
initializeImageMatcher(this.runner._config);
}
async run(entry: TestRunnerEntry) {
delete require.cache[entry.file];
const { TestRunner } = require('./testRunner');
const testRunner = new TestRunner(entry, this.runner._config, 0);
for (const event of ['testBegin', 'testStdOut', 'testStdErr', 'testEnd', 'done'])
testRunner.on(event, this.emit.bind(this, event));
testRunner.run();
}
async stop() {
await this.fixturePool.teardownScope('worker');
this.emit('exit');
}
}
function chunkFromParams(params: { testId: string, buffer?: string, text?: string }): string | Buffer {
if (typeof params.text === 'string')
return params.text;
return Buffer.from(params.buffer, 'base64');
}
function computeWorkerHash(file: string) {
// At this point, registrationsByFile contains all the files with worker fixture registrations.
// For every test, build the require closure and map each file to fixtures declared in it.
// This collection of fixtures is the fingerprint of the worker setup, a "worker hash".
// Tests with the matching "worker hash" will reuse the same worker.
const hash = crypto.createHash('sha1');
for (const registration of lookupRegistrations(file, 'worker').values())
hash.update(registration.location);
return hash.digest('hex');
}

32
test-runner/src/runnerConfig.ts
View file

@ -1,32 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type RunnerConfig = {
debug?: boolean;
forbidOnly?: boolean;
globalTimeout: number;
grep?: string;
jobs: number;
outputDir: string;
quiet?: boolean;
repeatEach: number;
retries: number,
snapshotDir: string;
testDir: string;
timeout: number;
trialRun?: boolean;
updateSnapshots?: boolean;
};

108
test-runner/src/spec.ts
View file

@ -1,108 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Test, Suite } from './test';
import { installTransform } from './transform';
Error.stackTraceLimit = 15;
function specBuilder(modifiers, specCallback) {
function builder(specs, last) {
const callable = (...args) => {
if (!last || (typeof args[0] === 'string' && typeof args[1] === 'function')) {
// Looks like a body (either it or describe). Assume that last modifier is true.
const newSpecs = { ...specs };
if (last)
newSpecs[last] = [true];
return specCallback(newSpecs, ...args);
}
const newSpecs = { ...specs };
newSpecs[last] = args;
return builder(newSpecs, null);
};
return new Proxy(callable, {
get: (obj, prop) => {
if (typeof prop === 'string' && modifiers.includes(prop)) {
const newSpecs = { ...specs };
// Modifier was not called, assume true.
if (last)
newSpecs[last] = [true];
return builder(newSpecs, prop);
}
return obj[prop];
},
});
}
return builder({}, null);
}
export function spec(suite: Suite, file: string, timeout: number): () => void {
const suites = [suite];
suite.file = file;
const it = specBuilder(['_skip', '_only'], (specs: any, title: string, metaFn: (test: Test) => void | Function, fn?: Function) => {
const suite = suites[0];
if (typeof fn !== 'function') {
fn = metaFn;
metaFn = null;
}
const test = new Test(title, fn);
if (metaFn)
metaFn(test);
test.file = file;
test._timeout = timeout;
const only = specs._only && specs._only[0];
if (only)
test._only = true;
if (!only && specs._skip && specs._skip[0])
test._skipped = true;
suite._addTest(test);
return test;
});
const describe = specBuilder(['_skip', '_only'], (specs: any, title: string, metaFn: (suite: Suite) => void | Function, fn?: Function) => {
if (typeof fn !== 'function') {
fn = metaFn;
metaFn = null;
}
const child = new Suite(title, suites[0]);
if (metaFn)
metaFn(child);
suites[0]._addSuite(child);
child.file = file;
const only = specs._only && specs._only[0];
if (only)
child._only = true;
if (!only && specs._skip && specs._skip[0])
child._skipped = true;
suites.unshift(child);
fn();
suites.shift();
});
(global as any).beforeEach = fn => suite._addHook('beforeEach', fn);
(global as any).afterEach = fn => suite._addHook('afterEach', fn);
(global as any).beforeAll = fn => suite._addHook('beforeAll', fn);
(global as any).afterAll = fn => suite._addHook('afterAll', fn);
(global as any).describe = describe;
(global as any).fdescribe = describe._only(true);
(global as any).xdescribe = describe._skip(true);
(global as any).it = it;
(global as any).fit = it._only(true);
(global as any).xit = it._skip(true);
return installTransform();
}

308
test-runner/src/test.ts
View file

@ -1,308 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type Configuration = { name: string, value: string }[];
type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
export class Runnable {
title: string;
file: string;
parent?: Suite;
_only = false;
_skipped = false;
_flaky = false;
_slow = false;
_expectedStatus: TestStatus = 'passed';
isOnly(): boolean {
return this._only;
}
isSlow(): boolean {
return this._slow;
}
slow(): void;
slow(condition: boolean): void;
slow(description: string): void;
slow(condition: boolean, description: string): void;
slow(arg?: boolean | string, description?: string) {
const { condition } = this._interpretCondition(arg, description);
if (condition)
this._slow = true;
}
skip(): void;
skip(condition: boolean): void;
skip(description: string): void;
skip(condition: boolean, description: string): void;
skip(arg?: boolean | string, description?: string) {
const { condition } = this._interpretCondition(arg, description);
if (condition)
this._skipped = true;
}
fixme(): void;
fixme(condition: boolean): void;
fixme(description: string): void;
fixme(condition: boolean, description: string): void;
fixme(arg?: boolean | string, description?: string) {
const { condition } = this._interpretCondition(arg, description);
if (condition)
this._skipped = true;
}
flaky(): void;
flaky(condition: boolean): void;
flaky(description: string): void;
flaky(condition: boolean, description: string): void;
flaky(arg?: boolean | string, description?: string) {
const { condition } = this._interpretCondition(arg, description);
if (condition)
this._flaky = true;
}
fail(): void;
fail(condition: boolean): void;
fail(description: string): void;
fail(condition: boolean, description: string): void;
fail(arg?: boolean | string, description?: string) {
const { condition } = this._interpretCondition(arg, description);
if (condition)
this._expectedStatus = 'failed';
}
private _interpretCondition(arg?: boolean | string, description?: string): { condition: boolean, description?: string } {
if (arg === undefined && description === undefined)
return { condition: true };
if (typeof arg === 'string')
return { condition: true, description: arg };
return { condition: !!arg, description };
}
_isSkipped(): boolean {
return this._skipped || (this.parent && this.parent._isSkipped());
}
_isSlow(): boolean {
return this._slow || (this.parent && this.parent._isSlow());
}
isFlaky(): boolean {
return this._flaky || (this.parent && this.parent.isFlaky());
}
titlePath(): string[] {
if (!this.parent)
return [];
return [...this.parent.titlePath(), this.title];
}
fullTitle(): string {
return this.titlePath().join(' ');
}
_copyFrom(other: Runnable) {
this.file = other.file;
this._only = other._only;
this._flaky = other._flaky;
this._skipped = other._skipped;
this._slow = other._slow;
}
}
export class Test extends Runnable {
fn: Function;
results: TestResult[] = [];
_id: string;
_overriddenFn: Function;
_startTime: number;
_timeout = 0;
constructor(title: string, fn: Function) {
super();
this.title = title;
this.fn = fn;
}
_appendResult(): TestResult {
const result: TestResult = {
duration: 0,
expectedStatus: 'passed',
stdout: [],
stderr: [],
data: {}
};
this.results.push(result);
return result;
}
timeout(): number {
return this._timeout;
}
_ok(): boolean {
if (this._isSkipped())
return true;
const hasFailedResults = !!this.results.find(r => r.status !== r.expectedStatus);
if (!hasFailedResults)
return true;
if (!this.isFlaky())
return false;
const hasPassedResults = !!this.results.find(r => r.status === r.expectedStatus);
return hasPassedResults;
}
_hasResultWithStatus(status: TestStatus): boolean {
return !!this.results.find(r => r.status === status);
}
_clone(): Test {
const test = new Test(this.title, this.fn);
test._copyFrom(this);
test._timeout = this._timeout;
test._overriddenFn = this._overriddenFn;
return test;
}
}
export type TestResult = {
duration: number;
status?: TestStatus;
expectedStatus: TestStatus;
error?: any;
stdout: (string | Buffer)[];
stderr: (string | Buffer)[];
data: any;
}
export class Suite extends Runnable {
suites: Suite[] = [];
tests: Test[] = [];
configuration: Configuration;
_configurationString: string;
_hooks: { type: string, fn: Function } [] = [];
_entries: (Suite | Test)[] = [];
constructor(title: string, parent?: Suite) {
super();
this.title = title;
this.parent = parent;
}
total(): number {
let count = 0;
this.findTest(fn => {
++count;
});
return count;
}
_addTest(test: Test) {
test.parent = this;
this.tests.push(test);
this._entries.push(test);
}
_addSuite(suite: Suite) {
suite.parent = this;
this.suites.push(suite);
this._entries.push(suite);
}
eachSuite(fn: (suite: Suite) => boolean | void): boolean {
for (const suite of this.suites) {
if (suite.eachSuite(fn))
return true;
}
return false;
}
findTest(fn: (test: Test) => boolean | void): boolean {
for (const suite of this.suites) {
if (suite.findTest(fn))
return true;
}
for (const test of this.tests) {
if (fn(test))
return true;
}
return false;
}
_clone(): Suite {
const suite = new Suite(this.title);
suite._copyFrom(this);
return suite;
}
_renumber() {
let ordinal = 0;
this.findTest((test: Test) => {
// All tests are identified with their ordinals.
test._id = `${ordinal++}@${this.file}::[${this._configurationString}]`;
});
}
_addHook(type: string, fn: any) {
this._hooks.push({ type, fn });
}
_hasTestsToRun(): boolean {
let found = false;
this.findTest(test => {
if (!test._isSkipped()) {
found = true;
return true;
}
});
return found;
}
}
export function serializeConfiguration(configuration: Configuration): string {
const tokens = [];
for (const { name, value } of configuration)
tokens.push(`${name}=${value}`);
return tokens.join(', ');
}
export function serializeError(error: Error | any): any {
if (error instanceof Error) {
return {
message: error.message,
stack: error.stack
};
}
return trimCycles(error);
}
function trimCycles(obj: any): any {
const cache = new Set();
return JSON.parse(
JSON.stringify(obj, function(key, value) {
if (typeof value === 'object' && value !== null) {
if (cache.has(value))
return '' + value;
cache.add(value);
}
return value;
})
);
}

139
test-runner/src/testCollector.ts
View file

@ -1,139 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import path from 'path';
import { fixturesForCallback } from './fixtures';
import { Test, Suite, serializeConfiguration } from './test';
import { spec } from './spec';
import { RunnerConfig } from './runnerConfig';
export type Matrix = {
[key: string]: string[]
};
export class TestCollector {
suite: Suite;
private _matrix: Matrix;
private _config: RunnerConfig;
private _grep: RegExp;
private _hasOnly: boolean;
constructor(files: string[], matrix: Matrix, config: RunnerConfig) {
this._matrix = matrix;
this._config = config;
this.suite = new Suite('');
if (config.grep) {
const match = config.grep.match(/^\/(.*)\/(g|i|)$|.*/);
this._grep = new RegExp(match[1] || match[0], match[2]);
}
for (const file of files)
this._addFile(file);
this._hasOnly = this._filterOnly(this.suite);
}
hasOnly() {
return this._hasOnly;
}
private _addFile(file: string) {
const suite = new Suite('');
const revertBabelRequire = spec(suite, file, this._config.timeout);
require(file);
revertBabelRequire();
const workerGeneratorConfigurations = new Map();
suite.findTest((test: Test) => {
// Get all the fixtures that the test needs.
const fixtures = fixturesForCallback(test.fn);
// For generator fixtures, collect all variants of the fixture values
// to build different workers for them.
const generatorConfigurations = [];
for (const name of fixtures) {
const values = this._matrix[name];
if (!values)
continue;
const state = generatorConfigurations.length ? generatorConfigurations.slice() : [[]];
generatorConfigurations.length = 0;
for (const gen of state) {
for (const value of values)
generatorConfigurations.push([...gen, { name, value }]);
}
}
// No generator fixtures for test, include empty set.
if (!generatorConfigurations.length)
generatorConfigurations.push([]);
for (const configuration of generatorConfigurations) {
// Serialize configuration as readable string, we will use it as a hash.
const configurationString = serializeConfiguration(configuration);
// Allocate worker for this configuration, add test into it.
if (!workerGeneratorConfigurations.has(configurationString))
workerGeneratorConfigurations.set(configurationString, { configuration, configurationString, tests: new Set() });
workerGeneratorConfigurations.get(configurationString).tests.add(test);
}
});
// Clone the suite as many times as we have repeat each.
for (let i = 0; i < this._config.repeatEach; ++i) {
// Clone the suite as many times as there are worker hashes.
// Only include the tests that requested these generations.
for (const [hash, {configuration, configurationString, tests}] of workerGeneratorConfigurations.entries()) {
const clone = this._cloneSuite(suite, tests);
this.suite._addSuite(clone);
clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : '') + ' ' + (i ? ` #repeat-${i}#` : '');
clone.configuration = configuration;
clone._configurationString = configurationString + `#repeat-${i}#`;
clone._renumber();
}
}
}
private _cloneSuite(suite: Suite, tests: Set<Test>) {
const copy = suite._clone();
for (const entry of suite._entries) {
if (entry instanceof Suite) {
copy._addSuite(this._cloneSuite(entry, tests));
} else {
const test = entry;
if (!tests.has(test))
continue;
if (this._grep && !this._grep.test(test.fullTitle()))
continue;
const testCopy = test._clone();
copy._addTest(testCopy);
}
}
return copy;
}
private _filterOnly(suite) {
const onlySuites = suite.suites.filter((child: Suite) => this._filterOnly(child) || child._only);
const onlyTests = suite.tests.filter((test: Test) => test._only);
if (onlySuites.length || onlyTests.length) {
suite.suites = onlySuites;
suite.tests = onlyTests;
return true;
}
return false;
}
}

234
test-runner/src/testRunner.ts
View file

@ -1,234 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FixturePool, rerunRegistrations, setParameters, TestInfo } from './fixtures';
import { EventEmitter } from 'events';
import { setCurrentTestFile } from './expect';
import { Test, Suite, Configuration, serializeError, TestResult } from './test';
import { spec } from './spec';
import { RunnerConfig } from './runnerConfig';
import * as util from 'util';
export const fixturePool = new FixturePool();
export type TestRunnerEntry = {
file: string;
ids: string[];
configurationString: string;
configuration: Configuration;
hash: string;
};
function chunkToParams(chunk: Buffer | string): { text?: string, buffer?: string } {
if (chunk instanceof Buffer)
return { buffer: chunk.toString('base64') };
if (typeof chunk !== 'string')
return { text: util.inspect(chunk) };
return { text: chunk };
}
export class TestRunner extends EventEmitter {
private _failedTestId: string | undefined;
private _fatalError: any | undefined;
private _ids: Set<string>;
private _remaining: Set<string>;
private _trialRun: any;
private _parsedGeneratorConfiguration: any = {};
private _config: RunnerConfig;
private _timeout: number;
private _testId: string | null;
private _stdOutBuffer: (string | Buffer)[] = [];
private _stdErrBuffer: (string | Buffer)[] = [];
private _testResult: TestResult | null = null;
private _suite: Suite;
private _loaded = false;
constructor(entry: TestRunnerEntry, config: RunnerConfig, workerId: number) {
super();
this._suite = new Suite('');
this._suite.file = entry.file;
this._suite._configurationString = entry.configurationString;
this._ids = new Set(entry.ids);
this._remaining = new Set(entry.ids);
this._trialRun = config.trialRun;
this._timeout = config.timeout;
this._config = config;
for (const {name, value} of entry.configuration)
this._parsedGeneratorConfiguration[name] = value;
this._parsedGeneratorConfiguration['parallelIndex'] = workerId;
setCurrentTestFile(this._suite.file);
}
stop() {
this._trialRun = true;
}
unhandledError(error: Error | any) {
if (this._testResult) {
this._testResult.status = 'failed';
this._testResult.error = serializeError(error);
this._failedTestId = this._testId;
this.emit('testEnd', {
id: this._testId,
result: this._testResult
});
this._testResult = null;
} else if (!this._loaded) {
// No current test - fatal error.
this._fatalError = serializeError(error);
}
this._reportDone();
}
stdout(chunk: string | Buffer) {
this._stdOutBuffer.push(chunk);
if (!this._testId)
return;
for (const c of this._stdOutBuffer)
this.emit('testStdOut', { id: this._testId, ...chunkToParams(c) });
this._stdOutBuffer = [];
}
stderr(chunk: string | Buffer) {
this._stdErrBuffer.push(chunk);
if (!this._testId)
return;
for (const c of this._stdErrBuffer)
this.emit('testStdErr', { id: this._testId, ...chunkToParams(c) });
this._stdErrBuffer = [];
}
async run() {
setParameters(this._parsedGeneratorConfiguration);
const revertBabelRequire = spec(this._suite, this._suite.file, this._timeout);
require(this._suite.file);
revertBabelRequire();
this._suite._renumber();
this._loaded = true;
rerunRegistrations(this._suite.file, 'test');
await this._runSuite(this._suite);
this._reportDone();
}
private async _runSuite(suite: Suite) {
try {
await this._runHooks(suite, 'beforeAll', 'before');
} catch (e) {
this._fatalError = serializeError(e);
this._reportDone();
}
for (const entry of suite._entries) {
if (entry instanceof Suite)
await this._runSuite(entry);
else
await this._runTest(entry);
}
try {
await this._runHooks(suite, 'afterAll', 'after');
} catch (e) {
this._fatalError = serializeError(e);
this._reportDone();
}
}
private async _runTest(test: Test) {
if (this._failedTestId)
return false;
if (this._ids.size && !this._ids.has(test._id))
return;
this._remaining.delete(test._id);
const id = test._id;
this._testId = id;
// We only know resolved skipped/flaky value in the worker,
// send it to the runner.
test._skipped = test._isSkipped();
test._flaky = test.isFlaky();
test._slow = test._isSlow();
this.emit('testBegin', {
id,
skipped: test._skipped,
flaky: test._flaky,
slow: test._slow
});
const result: TestResult = {
duration: 0,
status: 'passed',
expectedStatus: test._expectedStatus,
stdout: [],
stderr: [],
data: {}
};
this._testResult = result;
if (test._skipped) {
result.status = 'skipped';
this.emit('testEnd', { id, result });
return;
}
const startTime = Date.now();
try {
const testInfo = { config: this._config, test, result };
if (!this._trialRun) {
await this._runHooks(test.parent, 'beforeEach', 'before', testInfo);
const timeout = test._isSlow() ? this._timeout * 3 : this._timeout;
await fixturePool.runTestWithFixturesAndTimeout(test.fn, timeout, testInfo);
await this._runHooks(test.parent, 'afterEach', 'after', testInfo);
} else {
result.status = result.expectedStatus;
}
} catch (error) {
// Error in the test fixture teardown.
result.status = 'failed';
result.error = serializeError(error);
}
result.duration = Date.now() - startTime;
if (this._testResult) {
// We could have reported end due to an unhandled exception.
this.emit('testEnd', { id, result });
}
if (result.status !== 'passed')
this._failedTestId = this._testId;
this._testResult = null;
this._testId = null;
}
private async _runHooks(suite: Suite, type: string, dir: 'before' | 'after', testInfo?: TestInfo) {
if (!suite._hasTestsToRun())
return;
const all = [];
for (let s = suite; s; s = s.parent) {
const funcs = s._hooks.filter(e => e.type === type).map(e => e.fn);
all.push(...funcs.reverse());
}
if (dir === 'before')
all.reverse();
for (const hook of all)
await fixturePool.resolveParametersAndRun(hook, this._config, testInfo);
}
private _reportDone() {
this.emit('done', {
failedTestId: this._failedTestId,
fatalError: this._fatalError,
remaining: [...this._remaining],
});
}
}

77
test-runner/src/transform.ts
View file

@ -1,77 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as crypto from 'crypto';
import * as os from 'os';
import * as path from 'path';
import * as fs from 'fs';
import * as pirates from 'pirates';
import * as babel from '@babel/core';
import * as sourceMapSupport from 'source-map-support';
const version = 2;
const cacheDir = path.join(os.tmpdir(), 'playwright-transform-cache');
const sourceMaps: Map<string, string> = new Map();
sourceMapSupport.install({
environment: 'node',
handleUncaughtExceptions: false,
retrieveSourceMap(source) {
if (!sourceMaps.has(source))
return null;
const sourceMapPath = sourceMaps.get(source);
if (!fs.existsSync(sourceMapPath))
return null;
return {
map: JSON.parse(fs.readFileSync(sourceMapPath, 'utf-8')),
url: source
};
}
});
function calculateCachePath(content: string, filePath: string): string {
const hash = crypto.createHash('sha1').update(content).update(filePath).update(String(version)).digest('hex');
const fileName = path.basename(filePath, path.extname(filePath)).replace(/\W/g, '') + '_' + hash;
return path.join(cacheDir, hash[0] + hash[1], fileName);
}
export function installTransform(): () => void {
return pirates.addHook((code, filename) => {
const cachePath = calculateCachePath(code, filename);
const codePath = cachePath + '.js';
const sourceMapPath = cachePath + '.map';
sourceMaps.set(filename, sourceMapPath);
if (fs.existsSync(codePath))
return fs.readFileSync(codePath, 'utf8');
const result = babel.transformFileSync(filename, {
presets: [
['@babel/preset-env', { targets: {node: 'current'} }],
['@babel/preset-typescript', { onlyRemoveTypeImports: true }],
],
sourceMaps: true,
});
if (result.code) {
fs.mkdirSync(path.dirname(cachePath), {recursive: true});
if (result.map)
fs.writeFileSync(sourceMapPath, JSON.stringify(result.map), 'utf8');
fs.writeFileSync(codePath, result.code, 'utf8');
}
return result.code;
}, {
exts: ['.ts']
});
}

45
test-runner/src/util.ts
View file

@ -1,45 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export async function raceAgainstTimeout<T>(promise: Promise<T>, timeout: number): Promise<{ result?: T, timedOut?: boolean }> {
if (!timeout)
return { result: await promise };
let timer: NodeJS.Timer;
let done = false;
let fulfill: (t: { result?: T, timedOut?: boolean }) => void;
let reject: (e: Error) => void;
const result = new Promise((f, r) => {
fulfill = f;
reject = r;
});
setTimeout(() => {
done = true;
fulfill({ timedOut: true });
}, timeout);
promise.then(result => {
clearTimeout(timer);
if (!done) {
done = true;
fulfill({ result });
}
}).catch(e => {
clearTimeout(timer);
if (!done)
reject(e);
});
return result;
}

100
test-runner/src/worker.ts
View file

@ -1,100 +0,0 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { initializeImageMatcher } from './expect';
import { TestRunner, fixturePool } from './testRunner';
import { Console } from 'console';
let closed = false;
sendMessageToParent('ready');
global.console = new Console({
stdout: process.stdout,
stderr: process.stderr,
colorMode: process.env.FORCE_COLOR === '1',
});
process.stdout.write = chunk => {
if (testRunner)
testRunner.stdout(chunk);
return true;
};
process.stderr.write = chunk => {
if (testRunner)
testRunner.stderr(chunk);
return true;
};
process.on('disconnect', gracefullyCloseAndExit);
process.on('SIGINT',() => {});
process.on('SIGTERM',() => {});
let workerId: number;
let testRunner: TestRunner;
process.on('unhandledRejection', (reason, promise) => {
if (testRunner)
testRunner.unhandledError(reason);
});
process.on('uncaughtException', error => {
if (testRunner)
testRunner.unhandledError(error);
});
process.on('message', async message => {
if (message.method === 'init') {
workerId = message.params.workerId;
initializeImageMatcher(message.params);
return;
}
if (message.method === 'stop') {
await gracefullyCloseAndExit();
return;
}
if (message.method === 'run') {
testRunner = new TestRunner(message.params.entry, message.params.config, workerId);
for (const event of ['testBegin', 'testStdOut', 'testStdErr', 'testEnd', 'done'])
testRunner.on(event, sendMessageToParent.bind(null, event));
await testRunner.run();
testRunner = null;
}
});
async function gracefullyCloseAndExit() {
if (closed)
return;
closed = true;
// Force exit after 30 seconds.
setTimeout(() => process.exit(0), 30000);
// Meanwhile, try to gracefully close all browsers.
if (testRunner)
testRunner.stop();
await fixturePool.teardownScope('worker');
process.exit(0);
}
function sendMessageToParent(method, params = {}) {
if (closed)
return;
try {
process.send({ method, params });
} catch (e) {
// Can throw when closing.
}
}

30
test-runner/test/assets/allow-flaky.js
View file

@ -1,30 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const fs = require('fs');
const path = require('path');
it('flake', test => {
test.flaky();
}, async ({}) => {
try {
fs.readFileSync(path.join(__dirname, '..', 'test-results', 'allow-flaky.txt'));
} catch (e) {
// First time this fails.
fs.writeFileSync(path.join(__dirname, '..', 'test-results', 'allow-flaky.txt'), 'TRUE');
expect(true).toBe(false);
}
});

24
test-runner/test/assets/expected-failure.js
View file

@ -1,24 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('../../lib');
it('fails', test => test.fail(), () => {
expect(1 + 1).toBe(3);
});
it('non-empty remaining',() => {
expect(1 + 1).toBe(2);
});

30
test-runner/test/assets/fixture-timeout.js
View file

@ -1,30 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { registerFixture } = require('../../');
registerFixture('timeout', async ({}, runTest) => {
await runTest();
await new Promise(f => setTimeout(f, 100000));
});
it('fixture timeout', async({timeout}) => {
expect(1).toBe(1);
});
it('failing fixture timeout', async({timeout}) => {
expect(1).toBe(2);
});

4
test-runner/test/assets/global-foo.js
View file

@ -1,4 +0,0 @@
global.foo = true;
module.exports = {
abc: 123
};

24
test-runner/test/assets/nested-skip.js
View file

@ -1,24 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('../../');
describe('skipped', suite => {
suite.skip(true);
}, () => {
it('succeeds',() => {
expect(1 + 1).toBe(2);
});
});

20
test-runner/test/assets/one-failure.js
View file

@ -1,20 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('../../');
it('fails', () => {
expect(1 + 1).toBe(7);
});

20
test-runner/test/assets/one-success.js
View file

@ -1,20 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('../../');
it('succeeds', () => {
expect(1 + 1).toBe(2);
});

20
test-runner/test/assets/one-timeout.js
View file

@ -1,20 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('../../');
it('timeout', async () => {
await new Promise(f => setTimeout(f, 10000));
});

28
test-runner/test/assets/retry-failures.js
View file

@ -1,28 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const fs = require('fs');
const path = require('path');
it('flake', async ({}) => {
try {
fs.readFileSync(path.join(__dirname, '..', 'test-results', 'retry-failures.txt'));
} catch (e) {
// First time this fails.
fs.writeFileSync(path.join(__dirname, '..', 'test-results', 'retry-failures.txt'), 'TRUE');
expect(true).toBe(false);
}
});

24
test-runner/test/assets/stdio.js
View file

@ -1,24 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('../../..');
it('stdio', () => {
process.stdout.write('stdout text');
process.stdout.write(Buffer.from('stdout buffer'));
process.stderr.write('stderr text');
process.stderr.write(Buffer.from('stderr buffer'));
});

23
test-runner/test/assets/suite-error.js
View file

@ -1,23 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { parameters } = require('../../');
if (typeof parameters.parallelIndex === 'number')
throw new Error('Suite error');
it('passes',() => {
expect(1 + 1).toBe(2);
});

30
test-runner/test/assets/test-data-visible-in-fixture.js
View file

@ -1,30 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { registerFixture } = require('../../');
const fs = require('fs');
const path = require('path');
registerFixture('postProcess', async ({}, runTest, info) => {
await runTest('');
info.result.data['myname'] = 'myvalue';
});
it('ensure fixture handles test error', async ({ postProcess }) => {
console.log('console.log');
console.error('console.error');
expect(true).toBe(false);
});

29
test-runner/test/assets/test-error-visible-in-fixture.js
View file

@ -1,29 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { registerFixture } = require('../../');
const fs = require('fs');
const path = require('path');
registerFixture('postProcess', async ({}, runTest, info) => {
await runTest('');
const { config, result } = info;
fs.writeFileSync(path.join(config.outputDir, 'test-error-visible-in-fixture.txt'), JSON.stringify(result.error, undefined, 2));
});
it('ensure fixture handles test error', async ({ postProcess }) => {
expect(true).toBe(false);
});

26
test-runner/test/assets/typescript.ts
View file

@ -1,26 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import '../../';
import './global-foo';
it('should find global foo', () => {
expect(global['foo']).toBe(true);
});
it('should work with type annotations', () => {
const x: number = 5;
expect(x).toBe(5);
});

20
test-runner/test/assets/unexpected-pass.js
View file

@ -1,20 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('../../');
it('succeeds', test => test.fail(), () => {
expect(1 + 1).toBe(2);
});

23
test-runner/test/assets/unhandled-rejection.js
View file

@ -1,23 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
require('../../');
it('unhandled rejection', async () => {
setTimeout(() => {
throw new Error('Unhandled rejection in the test');
});
await new Promise(f => setTimeout(f, 20));
});

24
test-runner/test/assets/worker-fixture-error.js
View file

@ -1,24 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { registerWorkerFixture } = require('../../');
registerWorkerFixture('failure', async ({}, runTest) => {
throw new Error('Worker failed');
});
it('fails', async({failure}) => {
});

23
test-runner/test/assets/worker-fixture-timeout.js
View file

@ -1,23 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { registerWorkerFixture } = require('../../');
registerWorkerFixture('timeout', async ({}, runTest) => {
});
it('fails', async({timeout}) => {
});

223
test-runner/test/exit-code.spec.ts
View file

@ -1,223 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import colors from 'colors/safe';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import rimraf from 'rimraf';
import { promisify } from 'util';
import '../lib';
const removeFolderAsync = promisify(rimraf);
it('should fail', async () => {
const result = await runTest('one-failure.js');
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(1);
});
it('should timeout', async () => {
const { exitCode, passed, failed, timedOut } = await runTest('one-timeout.js', { timeout: 100 });
expect(exitCode).toBe(1);
expect(passed).toBe(0);
expect(failed).toBe(0);
expect(timedOut).toBe(1);
});
it('should succeed', async () => {
const result = await runTest('one-success.js');
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.failed).toBe(0);
});
it('should access error in fixture', async () => {
const result = await runTest('test-error-visible-in-fixture.js');
expect(result.exitCode).toBe(1);
const data = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-results', 'test-error-visible-in-fixture.txt')).toString());
expect(data.message).toContain('Object.is equality');
});
it('should access data in fixture', async () => {
const { exitCode, report } = await runTest('test-data-visible-in-fixture.js');
expect(exitCode).toBe(1);
const testResult = report.suites[0].tests[0].results[0];
expect(testResult.data).toEqual({ 'myname': 'myvalue' });
expect(testResult.stdout).toEqual([{ text: 'console.log\n' }]);
expect(testResult.stderr).toEqual([{ text: 'console.error\n' }]);
});
it('should handle fixture timeout', async () => {
const { exitCode, output, failed, timedOut } = await runTest('fixture-timeout.js', { timeout: 500 });
expect(exitCode).toBe(1);
expect(output).toContain('Timeout of 500ms');
expect(failed).toBe(1);
expect(timedOut).toBe(1);
});
it('should handle worker fixture timeout', async () => {
const result = await runTest('worker-fixture-timeout.js', { timeout: 500 });
expect(result.exitCode).toBe(1);
expect(result.output).toContain('Timeout of 500ms');
});
it('should handle worker fixture error', async () => {
const result = await runTest('worker-fixture-error.js');
expect(result.exitCode).toBe(1);
expect(result.output).toContain('Worker failed');
});
it('should collect stdio', async () => {
const { exitCode, report } = await runTest('stdio.js');
expect(exitCode).toBe(0);
const testResult = report.suites[0].tests[0].results[0];
const { stdout, stderr } = testResult;
expect(stdout).toEqual([{ text: 'stdout text' }, { buffer: Buffer.from('stdout buffer').toString('base64') }]);
expect(stderr).toEqual([{ text: 'stderr text' }, { buffer: Buffer.from('stderr buffer').toString('base64') }]);
});
it('should work with typescript', async () => {
const result = await runTest('typescript.ts');
expect(result.exitCode).toBe(0);
});
it('should retry failures', async () => {
const result = await runTest('retry-failures.js', { retries: 1 });
expect(result.exitCode).toBe(1);
expect(result.flaky).toBe(1);
});
it('should retry timeout', async () => {
const { exitCode, passed, failed, timedOut, output } = await runTest('one-timeout.js', { timeout: 100, retries: 2 });
expect(exitCode).toBe(1);
expect(passed).toBe(0);
expect(failed).toBe(0);
expect(timedOut).toBe(1);
expect(output.split('\n')[0]).toBe(colors.red('T').repeat(3));
});
it('should repeat each', async () => {
const { exitCode, report } = await runTest('one-success.js', { 'repeat-each': 3 });
expect(exitCode).toBe(0);
expect(report.suites.length).toBe(3);
for (const suite of report.suites)
expect(suite.tests.length).toBe(1);
});
it('should report suite errors', async () => {
const { exitCode, failed, output } = await runTest('suite-error.js');
expect(exitCode).toBe(1);
expect(failed).toBe(1);
expect(output).toContain('Suite error');
});
it('should allow flaky', async () => {
const result = await runTest('allow-flaky.js', { retries: 1 });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
});
it('should fail on unexpected pass', async () => {
const { exitCode, failed, output } = await runTest('unexpected-pass.js');
expect(exitCode).toBe(1);
expect(failed).toBe(1);
expect(output).toContain('passed unexpectedly');
});
it('should fail on unexpected pass with retries', async () => {
const { exitCode, failed, output } = await runTest('unexpected-pass.js', { retries: 1 });
expect(exitCode).toBe(1);
expect(failed).toBe(1);
expect(output).toContain('passed unexpectedly');
});
it('should not retry unexpected pass', async () => {
const { exitCode, passed, failed, output } = await runTest('unexpected-pass.js', { retries: 2 });
expect(exitCode).toBe(1);
expect(passed).toBe(0);
expect(failed).toBe(1);
expect(output.split('\n')[0]).toBe(colors.red('P'));
});
it('should not retry expected failure', async () => {
const { exitCode, passed, failed, output } = await runTest('expected-failure.js', { retries: 2 });
expect(exitCode).toBe(0);
expect(passed).toBe(2);
expect(failed).toBe(0);
expect(output.split('\n')[0]).toBe(colors.green('f') + colors.green('·'));
});
it('should respect nested skip', async () => {
const { exitCode, passed, failed, skipped } = await runTest('nested-skip.js');
expect(exitCode).toBe(0);
expect(passed).toBe(0);
expect(failed).toBe(0);
expect(skipped).toBe(1);
});
it('should retry unhandled rejection', async () => {
const result = await runTest('unhandled-rejection.js', { retries: 2 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.failed).toBe(1);
expect(result.output.split('\n')[0]).toBe(colors.red('F').repeat(3));
expect(result.output).toContain('Unhandled rejection');
});
it('should respect global timeout', async () => {
const { exitCode, output } = await runTest('one-timeout.js', { 'timeout': 100000, 'global-timeout': 500 });
expect(exitCode).toBe(1);
expect(output).toContain('Timed out waiting 0.5s for the entire test run');
});
async function runTest(filePath: string, params: any = {}) {
const outputDir = path.join(__dirname, 'test-results');
const reportFile = path.join(outputDir, 'results.json');
await removeFolderAsync(outputDir).catch(e => {});
const { output, status } = spawnSync('node', [
path.join(__dirname, '..', 'cli.js'),
path.join(__dirname, 'assets', filePath),
'--output=' + outputDir,
'--reporter=dot,json',
...Object.keys(params).map(key => `--${key}=${params[key]}`)
], {
env: {
...process.env,
PWRUNNER_JSON_REPORT: reportFile,
}
});
const passed = (/(\d+) passed/.exec(output.toString()) || [])[1];
const failed = (/(\d+) failed/.exec(output.toString()) || [])[1];
const timedOut = (/(\d+) timed out/.exec(output.toString()) || [])[1];
const flaky = (/(\d+) flaky/.exec(output.toString()) || [])[1];
const skipped = (/(\d+) skipped/.exec(output.toString()) || [])[1];
const report = JSON.parse(fs.readFileSync(reportFile).toString());
let outputStr = output.toString();
outputStr = outputStr.substring(1, outputStr.length - 1);
return {
exitCode: status,
output: outputStr,
passed: parseInt(passed, 10),
failed: parseInt(failed || '0', 10),
timedOut: parseInt(timedOut || '0', 10),
flaky: parseInt(flaky || '0', 10),
skipped: parseInt(skipped || '0', 10),
report
};
}

20
test-runner/tsconfig.json
View file

@ -1,20 +0,0 @@
{
"compilerOptions": {
"moduleResolution": "node",
"target": "ESNext",
"module": "commonjs",
"lib": ["esnext", "dom", "DOM.Iterable"],
"sourceMap": true,
"rootDir": "./src",
"outDir": "./lib",
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"declaration": true,
"allowJs": true,
"checkJs": true,
"declarationMap": true
},
"compileOnSave": true,
"include": ["src"],
"exclude": ["node_modules"]
}

2
test/chromium/oopif.spec.ts
View file

@ -15,7 +15,7 @@
*/
import { options } from '../playwright.fixtures';
import { registerWorkerFixture } from '../../test-runner';
import { registerWorkerFixture } from '@playwright/test-runner';
registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test) => {
const browser = await browserType.launch({

2
test/chromium/tracing.spec.ts
View file

@ -15,7 +15,7 @@
*/
import { options } from '../playwright.fixtures';
import { registerFixture } from '../../test-runner';
import { registerFixture } from '@playwright/test-runner';
import fs from 'fs';
import path from 'path';

2
test/downloads-path.spec.ts
View file

@ -17,7 +17,7 @@
import './playwright.fixtures';
import { registerFixture } from '../test-runner';
import { registerFixture } from '@playwright/test-runner';
import path from 'path';
import fs from 'fs';

2
test/electron/electron.fixture.ts
View file

@ -15,7 +15,7 @@
*/
import '../playwright.fixtures';
import { registerFixture } from '../../test-runner';
import { registerFixture } from '@playwright/test-runner';
import type {ElectronApplication, ElectronLauncher, ElectronPage} from '../../electron-types';
import path from 'path';

2
test/playwright.fixtures.ts
View file

@ -23,7 +23,7 @@ import { TestServer } from '../utils/testserver';
import { Connection } from '../lib/client/connection';
import { Transport } from '../lib/protocol/transport';
import { installCoverageHooks } from './coverage';
import { parameters, registerFixture, registerWorkerFixture } from '../test-runner';
import { parameters, registerFixture, registerWorkerFixture } from '@playwright/test-runner';
import { mkdtempAsync, removeFolderAsync } from './utils';
export const options = {

2
test/proxy.spec.ts
View file

@ -14,7 +14,7 @@
* limitations under the License.
*/
import { parameters } from '../test-runner';
import { parameters } from '@playwright/test-runner';
import { options } from './playwright.fixtures';
import socks from 'socksv5';

2
test/remoteServer.fixture.ts
View file

@ -14,7 +14,7 @@
* limitations under the License.
*/
import { registerFixture } from '../test-runner/lib';
import { registerFixture } from '@playwright/test-runner';
import path from 'path';
import { spawn } from 'child_process';

2
test/screencast.spec.ts
View file

@ -15,7 +15,7 @@
*/
import { options } from './playwright.fixtures';
import { registerFixture } from '../test-runner';
import { registerFixture } from '@playwright/test-runner';
import type { Page } from '..';
import fs from 'fs';

2
test/test-runner-helper.ts
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
import { registerFixture } from '../test-runner';
import { registerFixture } from '@playwright/test-runner';
declare global {
interface TestState {

2
test/test-runner-overrides-1.spec.ts
View file

@ -15,7 +15,7 @@
*/
import './test-runner-helper';
import { registerFixture } from '../test-runner';
import { registerFixture } from '@playwright/test-runner';
registerFixture('helperFixture', async ({}, test) => {
await test('helperFixture - overridden');

2
test/test-runner.spec.ts
View file

@ -15,7 +15,7 @@
*/
import './test-runner-helper';
import { registerFixture } from '../test-runner';
import { registerFixture } from '@playwright/test-runner';
declare global {
interface TestState {

2
utils/doclint/check_public_api/test/test.js
View file

@ -20,7 +20,7 @@ const checkPublicAPI = require('..');
const Source = require('../../Source');
const mdBuilder = require('../MDBuilder');
const jsBuilder = require('../JSBuilder');
const { registerWorkerFixture } = require('../../../../test-runner');
const { registerWorkerFixture } = require('@playwright/test-runner');
registerWorkerFixture('page', async({}, test) => {
const browser = await playwright.chromium.launch();

1
utils/watch.js
View file

@ -21,7 +21,6 @@ const fs = require('fs');
const spawns = [
child_process.spawn('node', [path.join(__dirname, 'runWebpack.js'), '--mode="development"', '--watch', '--silent'], { stdio: 'inherit', shell: true }),
child_process.spawn('npx', ['tsc', '-w', '--preserveWatchOutput', '-p', path.join(__dirname, '..')], { stdio: 'inherit', shell: true }),
child_process.spawn('npx', ['tsc', '-w', '--preserveWatchOutput', '-p', path.join(__dirname, '..', 'test-runner')], { stdio: 'inherit', shell: true }),
];
process.on('exit', () => spawns.forEach(s => s.kill()));