chore(testrunner): add exit code tests (#3562)

package.json

@@ -17,7 +17,7 @@
     "tsc": "tsc -p .",
     "tsc-installer": "tsc -p ./src/install/tsconfig.json",
     "doc": "node utils/doclint/cli.js",
-    "test-infra": "node test-runner/cli utils/doclint/check_public_api/test/test.js && node test-runner/cli utils/doclint/preprocessor/test.js",
+    "test-infra": "node test-runner/cli utils/doclint/check_public_api/test/test.js && node test-runner/cli utils/doclint/preprocessor/test.js && npm run test-testrunner",
     "lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && npm run generate-channels && npm run test-types && npm run test-infra",
     "clean": "rimraf lib && rimraf types",
     "prepare": "node install-from-github.js",
@@ -31,7 +31,8 @@
     "coverage": "node test/checkCoverage.js",
     "check-deps": "node utils/check_deps.js",
     "build-driver": "pkg --public --targets node12-linux-x64,node12-macos-x64,node12-win-x64 --out-path=drivers packages/playwright-driver/main.js",
-    "build-testrunner": "tsc -p test-runner"
+    "build-testrunner": "tsc -p test-runner",
+    "test-testrunner": "node test-runner/cli test-runner/test"
   },
   "author": {
     "name": "Microsoft Corporation"

test-runner/src/cli.js

@@ -43,7 +43,7 @@ program
   .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
   .action(async (command) => {
     // Collect files]
-    const testDir = path.join(process.cwd(), command.args[0]);
+    const testDir = path.resolve(process.cwd(), command.args[0]);
     const files = collectFiles(testDir, '', command.args.slice(1));
 
     const revertBabelRequire = installTransform();

test-runner/test/assets/one-failure.js

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../');
+
+it('fails',() => {
+  expect(1 + 1).toBe(7);
+});

test-runner/test/assets/one-success.js

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../');
+
+it('succeeds',() => {
+  expect(1 + 1).toBe(2);
+});

test-runner/test/exit-code.spec.ts

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import "../lib"
+import { spawnSync } from "child_process";
+import path from 'path';
+
+it('should fail', async() => {
+  const result = runTest('one-failure.js');
+  expect(result.exitCode).toBe(1);
+  expect(result.passing).toBe(0);
+  expect(result.failing).toBe(1);
+});
+
+it('should succeed', async() => {
+  const result = runTest('one-success.js');
+  expect(result.exitCode).toBe(0);
+  expect(result.passing).toBe(1);
+  expect(result.failing).toBe(0);
+});
+
+function runTest(filePath: string) {
+  const {output, status} = spawnSync('node', [path.join(__dirname, '..', 'cli.js'), path.join(__dirname, 'assets', filePath)]);
+  const passing = (/  (\d+) passing/.exec(output.toString()) || [])[1];
+  const failing = (/  (\d+) failing/.exec(output.toString()) || [])[1];
+  return {
+    exitCode: status,
+    output,
+    passing: parseInt(passing),
+    failing: parseInt(failing || '0')
+  }
+}