From d4dac0421228666f49cd303966e88833bd73e446 Mon Sep 17 00:00:00 2001 From: Joel Einbinder Date: Fri, 21 Aug 2020 13:22:43 -0700 Subject: [PATCH] chore(testrunner): add exit code tests (#3562) --- package.json | 5 +-- test-runner/src/cli.js | 2 +- test-runner/test/assets/one-failure.js | 20 ++++++++++++ test-runner/test/assets/one-success.js | 20 ++++++++++++ test-runner/test/exit-code.spec.ts | 44 ++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 test-runner/test/assets/one-failure.js create mode 100644 test-runner/test/assets/one-success.js create mode 100644 test-runner/test/exit-code.spec.ts diff --git a/package.json b/package.json index 2f54b47209..99d7cd5ade 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "tsc": "tsc -p .", "tsc-installer": "tsc -p ./src/install/tsconfig.json", "doc": "node utils/doclint/cli.js", - "test-infra": "node test-runner/cli utils/doclint/check_public_api/test/test.js && node test-runner/cli utils/doclint/preprocessor/test.js", + "test-infra": "node test-runner/cli utils/doclint/check_public_api/test/test.js && node test-runner/cli utils/doclint/preprocessor/test.js && npm run test-testrunner", "lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra", "clean": "rimraf lib && rimraf types", "prepare": "node install-from-github.js", @@ -31,7 +31,8 @@ "coverage": "node test/checkCoverage.js", "check-deps": "node utils/check_deps.js", "build-driver": "pkg --public --targets node12-linux-x64,node12-macos-x64,node12-win-x64 --out-path=drivers packages/playwright-driver/main.js", - "build-testrunner": "tsc -p test-runner" + "build-testrunner": "tsc -p test-runner", + "test-testrunner": "node test-runner/cli test-runner/test" }, "author": { "name": "Microsoft Corporation" diff --git a/test-runner/src/cli.js b/test-runner/src/cli.js index f168416c0b..0df874a0d6 100644 --- a/test-runner/src/cli.js +++ b/test-runner/src/cli.js @@ -43,7 +43,7 @@ program .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run') .action(async (command) => { // Collect files] - const testDir = path.join(process.cwd(), command.args[0]); + const testDir = path.resolve(process.cwd(), command.args[0]); const files = collectFiles(testDir, '', command.args.slice(1)); const revertBabelRequire = installTransform(); diff --git a/test-runner/test/assets/one-failure.js b/test-runner/test/assets/one-failure.js new file mode 100644 index 0000000000..a74fb8e15d --- /dev/null +++ b/test-runner/test/assets/one-failure.js @@ -0,0 +1,20 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +require('../../'); + +it('fails',() => { + expect(1 + 1).toBe(7); +}); diff --git a/test-runner/test/assets/one-success.js b/test-runner/test/assets/one-success.js new file mode 100644 index 0000000000..e3fec57a76 --- /dev/null +++ b/test-runner/test/assets/one-success.js @@ -0,0 +1,20 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +require('../../'); + +it('succeeds',() => { + expect(1 + 1).toBe(2); +}); diff --git a/test-runner/test/exit-code.spec.ts b/test-runner/test/exit-code.spec.ts new file mode 100644 index 0000000000..72a36c567a --- /dev/null +++ b/test-runner/test/exit-code.spec.ts @@ -0,0 +1,44 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import "../lib" +import { spawnSync } from "child_process"; +import path from 'path'; + +it('should fail', async() => { + const result = runTest('one-failure.js'); + expect(result.exitCode).toBe(1); + expect(result.passing).toBe(0); + expect(result.failing).toBe(1); +}); + +it('should succeed', async() => { + const result = runTest('one-success.js'); + expect(result.exitCode).toBe(0); + expect(result.passing).toBe(1); + expect(result.failing).toBe(0); +}); + +function runTest(filePath: string) { + const {output, status} = spawnSync('node', [path.join(__dirname, '..', 'cli.js'), path.join(__dirname, 'assets', filePath)]); + const passing = (/ (\d+) passing/.exec(output.toString()) || [])[1]; + const failing = (/ (\d+) failing/.exec(output.toString()) || [])[1]; + return { + exitCode: status, + output, + passing: parseInt(passing), + failing: parseInt(failing || '0') + } +} \ No newline at end of file