diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 572898bc6f..2f2ce2b8a4 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -40,8 +40,6 @@ jobs:
     - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json && npm run coverage"
       env:
         BROWSER: ${{ matrix.browser }}
-        DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
-        DEBUG_FILE: "test-results/debug.log"
         PWRUNNER_JSON_REPORT: "test-results/report.json"
     - uses: actions/upload-artifact@v1
       if: always()
@@ -67,8 +65,6 @@ jobs:
     - run: node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json
       env:
         BROWSER: ${{ matrix.browser }}
-        DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
-        DEBUG_FILE: "test-results/debug.log"
         PWRUNNER_JSON_REPORT: "test-results/report.json"
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
@@ -98,8 +94,6 @@ jobs:
       shell: bash
       env:
         BROWSER: ${{ matrix.browser }}
-        DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
-        DEBUG_FILE: "test-results/debug.log"
         PWRUNNER_JSON_REPORT: "test-results/report.json"
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
@@ -152,7 +146,6 @@ jobs:
       env:
         BROWSER: ${{ matrix.browser }}
         HEADLESS: "false"
-        DEBUG_FILE: "test-results/debug.log"
         PWRUNNER_JSON_REPORT: "test-results/report.json"
     - uses: actions/upload-artifact@v1
       if: ${{ always() }}
@@ -184,8 +177,6 @@ jobs:
     - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && node test-runner/cli test/ --jobs=1 --forbid-only --timeout=30000 --retries=3 --reporter=dot,json"
       env:
         BROWSER: ${{ matrix.browser }}
-        DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
-        DEBUG_FILE: "test-results/debug.log"
         PWWIRE: true
         PWRUNNER_JSON_REPORT: "test-results/report.json"
     - uses: actions/upload-artifact@v1
diff --git a/test-runner/src/builtin.fixtures.ts b/test-runner/src/builtin.fixtures.ts
index 0e59bbcda1..83d3d1c856 100644
--- a/test-runner/src/builtin.fixtures.ts
+++ b/test-runner/src/builtin.fixtures.ts
@@ -32,6 +32,7 @@ declare global {
 
   type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void> | void) => void) & {
     fail(condition: boolean): ItFunction<STATE>;
+    fixme(condition: boolean): ItFunction<STATE>;
     flaky(condition: boolean): ItFunction<STATE>;
     skip(condition: boolean): ItFunction<STATE>;
     slow(): ItFunction<STATE>;
diff --git a/test-runner/src/reporters/base.ts b/test-runner/src/reporters/base.ts
index 01e7095d8d..eee022e09d 100644
--- a/test-runner/src/reporters/base.ts
+++ b/test-runner/src/reporters/base.ts
@@ -30,10 +30,9 @@ const stackUtils = new StackUtils();
 
 export class BaseReporter implements Reporter  {
   skipped: Test[] = [];
-  passed: Test[] = [];
+  asExpected: Test[] = [];
+  unexpected = new Set<Test>();
   flaky: Test[] = [];
-  failed: Test[] = [];
-  timedOut: Test[] = [];
   duration = 0;
   startTime: number;
   config: RunnerConfig;
@@ -66,28 +65,24 @@ export class BaseReporter implements Reporter  {
   }
 
   onTestEnd(test: Test, result: TestResult) {
-    switch (result.status) {
-      case 'skipped': {
-        this.skipped.push(test);
-        return;
-      }
-      case 'passed':
-        if (test.results.length === 1)
-          this.passed.push(test);
-        else
-          this.flaky.push(test);
-        return;
-      case 'failed':
-        // Fall through.
-      case 'timedOut': {
-        if (test.results.length === this.config.retries + 1) {
-          if (result.status === 'timedOut')
-            this.timedOut.push(test);
-          else
-            this.failed.push(test);
-        }
-        return;
+    if (result.status === 'skipped') {
+      this.skipped.push(test);
+      return;
+    }
+
+    if (result.status === result.expectedStatus) {
+      if (test.results.length === 1) {
+        // as expected from the first attempt
+        this.asExpected.push(test);
+      } else {
+        // as expected after unexpected -> flaky.
+        this.flaky.push(test);
       }
+      return;
+    }
+    if (result.status === 'passed' || result.status === 'timedOut' || test.results.length === this.config.retries + 1) {
+      // We made as many retries as we could, still failing.
+      this.unexpected.add(test);
     }
   }
 
@@ -98,15 +93,16 @@ export class BaseReporter implements Reporter  {
   epilogue() {
     console.log('');
 
-    console.log(colors.green(`  ${this.passed.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
+    console.log(colors.green(`  ${this.asExpected.length} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
 
     if (this.skipped.length)
       console.log(colors.yellow(`  ${this.skipped.length} skipped`));
 
-    if (this.failed.length) {
-      console.log(colors.red(`  ${this.failed.length} failed`));
+    const filteredUnexpected = [...this.unexpected].filter(t => !t._hasResultWithStatus('timedOut'));
+    if (filteredUnexpected.length) {
+      console.log(colors.red(`  ${filteredUnexpected.length} failed`));
       console.log('');
-      this._printFailures(this.failed);
+      this._printFailures(filteredUnexpected);
     }
 
     if (this.flaky.length) {
@@ -115,11 +111,13 @@ export class BaseReporter implements Reporter  {
       this._printFailures(this.flaky);
     }
 
-    if (this.timedOut.length) {
-      console.log(colors.red(`  ${this.timedOut.length} timed out`));
+    const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut'));
+    if (timedOut.length) {
+      console.log(colors.red(`  ${timedOut.length} timed out`));
       console.log('');
-      this._printFailures(this.timedOut);
+      this._printFailures(timedOut);
     }
+    console.log('');
   }
 
   private _printFailures(failures: Test[]) {
@@ -131,7 +129,8 @@ export class BaseReporter implements Reporter  {
   formatFailure(test: Test, index?: number): string {
     const tokens: string[] = [];
     const relativePath = path.relative(process.cwd(), test.file);
-    const header = `  ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}`;
+    const passedUnexpectedlySuffix = test.results[0].status === 'passed' ? ' -- passed unexpectedly' : '';
+    const header = `  ${index ? index + ')' : ''} ${terminalLink(relativePath, `file://${os.hostname()}${test.file}`)} › ${test.title}${passedUnexpectedlySuffix}`;
     tokens.push(colors.bold(colors.red(header)));
     for (const result of test.results) {
       if (result.status === 'passed')
diff --git a/test-runner/src/reporters/dot.ts b/test-runner/src/reporters/dot.ts
index dc98f00904..b065ec1e1e 100644
--- a/test-runner/src/reporters/dot.ts
+++ b/test-runner/src/reporters/dot.ts
@@ -23,8 +23,8 @@ class DotReporter extends BaseReporter {
     super.onTestEnd(test, result);
     switch (result.status) {
       case 'skipped': process.stdout.write(colors.yellow('∘')); break;
-      case 'passed': process.stdout.write(colors.green('·')); break;
-      case 'failed': process.stdout.write(colors.red(test.results.length > 1 ? '' + test.results.length : 'F')); break;
+      case 'passed': process.stdout.write(result.status === result.expectedStatus ? colors.green('·') : colors.red('P')); break;
+      case 'failed': process.stdout.write(result.status === result.expectedStatus ? colors.green('f') : colors.red('F')); break;
       case 'timedOut': process.stdout.write(colors.red('T')); break;
     }
   }
diff --git a/test-runner/src/reporters/list.ts b/test-runner/src/reporters/list.ts
index ab37cdc6d5..4d485cd184 100644
--- a/test-runner/src/reporters/list.ts
+++ b/test-runner/src/reporters/list.ts
@@ -35,12 +35,14 @@ class ListReporter extends BaseReporter {
   onTestEnd(test: Test, result: TestResult) {
     super.onTestEnd(test, result);
     let text = '';
-    switch (result.status) {
-      case 'skipped': text = colors.green('  - ') + colors.cyan(test.fullTitle()); break;
-      case 'passed': text = '\u001b[2K\u001b[0G' + colors.green('  ✓ ') + colors.gray(test.fullTitle()); break;
-      case 'failed':
-        // fall through
-      case 'timedOut': text = '\u001b[2K\u001b[0G' + colors.red(`  ${++this._failure}) ` + test.fullTitle()); break;
+    if (result.status === 'skipped') {
+      text = colors.green('  - ') + colors.cyan(test.fullTitle());
+    } else {
+      const statusMark = result.status === 'passed' ? colors.green('  ✓ ') : colors.red('  x ');
+      if (result.status === result.expectedStatus)
+        text = '\u001b[2K\u001b[0G' + statusMark + colors.gray(test.fullTitle());
+      else
+        text = '\u001b[2K\u001b[0G' + colors.red(`  ${++this._failure}) ` + test.fullTitle());
     }
     process.stdout.write(text + '\n');
   }
diff --git a/test-runner/src/reporters/pytest.ts b/test-runner/src/reporters/pytest.ts
index 9e4debb979..6e967085ef 100644
--- a/test-runner/src/reporters/pytest.ts
+++ b/test-runner/src/reporters/pytest.ts
@@ -149,14 +149,15 @@ class PytestReporter extends BaseReporter {
     }
 
     const status = [];
-    if (this.passed.length)
-      status.push(colors.green(`${this.passed.length} passed`));
+    if (this.asExpected.length)
+      status.push(colors.green(`${this.asExpected.length} as expected`));
     if (this.skipped.length)
       status.push(colors.yellow(`${this.skipped.length} skipped`));
-    if (this.failed.length)
-      status.push(colors.red(`${this.failed.length} failed`));
-    if (this.timedOut.length)
-      status.push(colors.red(`${this.timedOut.length} timed out`));
+    const timedOut = [...this.unexpected].filter(t => t._hasResultWithStatus('timedOut'));
+    if (this.unexpected.size - timedOut.length)
+      status.push(colors.red(`${this.unexpected.size - timedOut.length} unexpected failures`));
+    if (timedOut.length)
+      status.push(colors.red(`${timedOut.length} timed out`));
     status.push(colors.dim(`(${milliseconds(Date.now() - this.startTime)})`));
 
     for (let i = lines.length; i < this._visibleRows; ++i)
diff --git a/test-runner/src/runner.ts b/test-runner/src/runner.ts
index 65571b6edc..cdd2093af9 100644
--- a/test-runner/src/runner.ts
+++ b/test-runner/src/runner.ts
@@ -103,7 +103,11 @@ export class Runner {
     let doneCallback;
     const result = new Promise(f => doneCallback = f);
     worker.once('done', params => {
-      if (!params.failedTestId && !params.fatalError) {
+      // We won't file remaining if:
+      // - there are no remaining
+      // - we are here not because something failed
+      // - no unrecoverable worker error
+      if (!params.remaining.length && !params.failedTestId && !params.fatalError) {
         this._workerAvailable(worker);
         doneCallback();
         return;
@@ -112,8 +116,8 @@ export class Runner {
       // When worker encounters error, we will restart it.
       this._restartWorker(worker);
 
-      // In case of fatal error without test id, we are done with the entry.
-      if (params.fatalError && !params.failedTestId) {
+      // In case of fatal error, we are done with the entry.
+      if (params.fatalError) {
         // Report all the tests are failing with this error.
         for (const id of entry.ids) {
           const { test, result } = this._testById.get(id);
@@ -127,13 +131,16 @@ export class Runner {
       }
 
       const remaining = params.remaining;
-      if (this._config.retries) {
+
+      // Only retry expected failures, not passes and only if the test failed.
+      if (this._config.retries && params.failedTestId) {
         const pair = this._testById.get(params.failedTestId);
-        if (pair.test.results.length < this._config.retries + 1) {
+        if (pair.result.expectedStatus === 'passed' && pair.test.results.length < this._config.retries + 1) {
           pair.result = pair.test._appendResult();
           remaining.unshift(pair.test._id);
         }
       }
+
       if (remaining.length)
         this._queue.unshift({ ...entry, ids: remaining });
 
@@ -169,6 +176,7 @@ export class Runner {
       const { test } = this._testById.get(params.id);
       test._skipped = params.skipped;
       test._flaky = params.flaky;
+      test._expectedStatus = params.expectedStatus;
       this._reporter.onTestBegin(test);
     });
     worker.on('testEnd', params => {
diff --git a/test-runner/src/spec.ts b/test-runner/src/spec.ts
index bcdbeecd11..f9a70a29ac 100644
--- a/test-runner/src/spec.ts
+++ b/test-runner/src/spec.ts
@@ -53,7 +53,7 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
   const suites = [suite];
   suite.file = file;
 
-  const it = specBuilder(['skip', 'fail', 'slow', 'only', 'flaky'], (specs, title, fn) => {
+  const it = specBuilder(['skip', 'fixme', 'fail', 'slow', 'only', 'flaky'], (specs, title, fn) => {
     const suite = suites[0];
     const test = new Test(title, fn);
     test.file = file;
@@ -65,8 +65,10 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
       test.only = true;
     if (!only && specs.skip && specs.skip[0])
       test._skipped = true;
-    if (!only && specs.fail && specs.fail[0])
+    if (!only && specs.fixme && specs.fixme[0])
       test._skipped = true;
+    if (specs.fail && specs.fail[0])
+      test._expectedStatus = 'failed';
     if (specs.flaky && specs.flaky[0])
       test._flaky = true;
     suite._addTest(test);
@@ -81,7 +83,9 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
     if (only)
       child.only = true;
     if (!only && specs.skip && specs.skip[0])
-      child.skipped = true;
+      child._skipped = true;
+    if (!only && specs.fixme && specs.fixme[0])
+      child._skipped = true;
     suites.unshift(child);
     fn();
     suites.shift();
diff --git a/test-runner/src/test.ts b/test-runner/src/test.ts
index 70a36fa5ca..4a681dcef1 100644
--- a/test-runner/src/test.ts
+++ b/test-runner/src/test.ts
@@ -16,6 +16,8 @@
 
 export type Configuration = { name: string, value: string }[];
 
+type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped';
+
 export class Test {
   suite: Suite;
   title: string;
@@ -27,10 +29,13 @@ export class Test {
   results: TestResult[] = [];
 
   _id: string;
+  // Skipped & flaky are resolved based on options in worker only
+  // We will compute them there and send to the runner (front-end)
   _skipped = false;
   _flaky = false;
   _overriddenFn: Function;
   _startTime: number;
+  _expectedStatus: TestStatus = 'passed';
 
   constructor(title: string, fn: Function) {
     this.title = title;
@@ -48,7 +53,7 @@ export class Test {
   _appendResult(): TestResult {
     const result: TestResult = {
       duration: 0,
-      status: 'none',
+      expectedStatus: 'passed',
       stdout: [],
       stderr: [],
       data: {}
@@ -58,17 +63,21 @@ export class Test {
   }
 
   _ok(): boolean {
-    if (this._skipped)
+    if (this._skipped || this.suite._isSkipped())
       return true;
-    const hasFailedResults = !!this.results.find(r => r.status !== 'passed' && r.status !== 'skipped');
+    const hasFailedResults = !!this.results.find(r => r.status !== r.expectedStatus);
     if (!hasFailedResults)
       return true;
     if (!this._flaky)
       return false;
-    const hasPassedResults = !!this.results.find(r => r.status === 'passed');
+    const hasPassedResults = !!this.results.find(r => r.status === r.expectedStatus);
     return hasPassedResults;
   }
 
+  _hasResultWithStatus(status: TestStatus): boolean {
+    return !!this.results.find(r => r.status === status);
+  }
+
   _clone(): Test {
     const test = new Test(this.title, this.fn);
     test.suite = this.suite;
@@ -83,7 +92,8 @@ export class Test {
 
 export type TestResult = {
   duration: number;
-  status: 'none' | 'passed' | 'failed' | 'timedOut' | 'skipped';
+  status?: TestStatus;
+  expectedStatus: TestStatus;
   error?: any;
   stdout: (string | Buffer)[];
   stderr: (string | Buffer)[];
@@ -96,9 +106,12 @@ export class Suite {
   suites: Suite[] = [];
   tests: Test[] = [];
   only = false;
-  skipped = false;
   file: string;
   configuration: Configuration;
+
+  // Skipped & flaky are resolved based on options in worker only
+  // We will compute them there and send to the runner (front-end)
+  _skipped = false;
   _configurationString: string;
 
   _hooks: { type: string, fn: Function } [] = [];
@@ -124,7 +137,7 @@ export class Suite {
   }
 
   _isSkipped(): boolean {
-    return this.skipped || (this.parent && this.parent._isSkipped());
+    return this._skipped || (this.parent && this.parent._isSkipped());
   }
 
   _addTest(test: Test) {
@@ -163,7 +176,7 @@ export class Suite {
     const suite = new Suite(this.title);
     suite.only = this.only;
     suite.file = this.file;
-    suite.skipped = this.skipped;
+    suite._skipped = this._skipped;
     return suite;
   }
 
diff --git a/test-runner/src/testRunner.ts b/test-runner/src/testRunner.ts
index 2f61e1ff3a..d5d979190c 100644
--- a/test-runner/src/testRunner.ts
+++ b/test-runner/src/testRunner.ts
@@ -46,7 +46,6 @@ export class TestRunner extends EventEmitter {
   private _ids: Set<string>;
   private _remaining: Set<string>;
   private _trialRun: any;
-  private _configuredFile: any;
   private _parsedGeneratorConfiguration: any = {};
   private _config: RunnerConfig;
   private _timeout: number;
@@ -55,6 +54,7 @@ export class TestRunner extends EventEmitter {
   private _stdErrBuffer: (string | Buffer)[] = [];
   private _testResult: TestResult | null = null;
   private _suite: Suite;
+  private _loaded = false;
 
   constructor(entry: TestRunnerEntry, config: RunnerConfig, workerId: number) {
     super();
@@ -66,7 +66,6 @@ export class TestRunner extends EventEmitter {
     this._trialRun = config.trialRun;
     this._timeout = config.timeout;
     this._config = config;
-    this._configuredFile = entry.file + `::[${entry.configurationString}]`;
     for (const {name, value} of entry.configuration)
       this._parsedGeneratorConfiguration[name] = value;
     this._parsedGeneratorConfiguration['parallelIndex'] = workerId;
@@ -77,14 +76,16 @@ export class TestRunner extends EventEmitter {
     this._trialRun = true;
   }
 
-  fatalError(error: Error | any) {
-    this._fatalError = serializeError(error);
+  unhandledError(error: Error | any) {
     if (this._testResult) {
-      this._testResult.error = this._fatalError;
+      this._testResult.error = serializeError(error);
       this.emit('testEnd', {
         id: this._testId,
         result: this._testResult
       });
+    } else if (!this._loaded) {
+      // No current test - fatal error.
+      this._fatalError = serializeError(error);
     }
     this._reportDone();
   }
@@ -114,6 +115,7 @@ export class TestRunner extends EventEmitter {
     require(this._suite.file);
     revertBabelRequire();
     this._suite._renumber();
+    this._loaded = true;
 
     rerunRegistrations(this._suite.file, 'test');
     await this._runSuite(this._suite);
@@ -132,7 +134,6 @@ export class TestRunner extends EventEmitter {
         await this._runSuite(entry);
       else
         await this._runTest(entry);
-
     }
     try {
       await this._runHooks(suite, 'afterAll', 'after');
@@ -151,6 +152,9 @@ export class TestRunner extends EventEmitter {
 
     const id = test._id;
     this._testId = id;
+    // We only know resolved skipped/flaky value in the worker,
+    // send it to the runner.
+    test._skipped = test._skipped || test.suite._isSkipped();
     this.emit('testBegin', {
       id,
       skipped: test._skipped,
@@ -160,13 +164,14 @@ export class TestRunner extends EventEmitter {
     const result: TestResult = {
       duration: 0,
       status: 'passed',
+      expectedStatus: test._expectedStatus,
       stdout: [],
       stderr: [],
       data: {}
     };
     this._testResult = result;
 
-    if (test._skipped || test.suite._isSkipped()) {
+    if (test._skipped) {
       result.status = 'skipped';
       this.emit('testEnd', { id, result });
       return;
@@ -181,7 +186,7 @@ export class TestRunner extends EventEmitter {
         await fixturePool.runTestWithFixtures(test.fn, timeout, testInfo);
         await this._runHooks(test.suite, 'afterEach', 'after', testInfo);
       } else {
-        result.status = 'passed';
+        result.status = result.expectedStatus;
       }
     } catch (error) {
       // Error in the test fixture teardown.
diff --git a/test-runner/src/worker.ts b/test-runner/src/worker.ts
index fbe2d1479e..1d72ce72d9 100644
--- a/test-runner/src/worker.ts
+++ b/test-runner/src/worker.ts
@@ -49,12 +49,12 @@ let testRunner: TestRunner;
 
 process.on('unhandledRejection', (reason, promise) => {
   if (testRunner)
-    testRunner.fatalError(reason);
+    testRunner.unhandledError(reason);
 });
 
 process.on('uncaughtException', error => {
   if (testRunner)
-    testRunner.fatalError(error);
+    testRunner.unhandledError(error);
 });
 
 process.on('message', async message => {
diff --git a/test-runner/test/assets/expected-failure.js b/test-runner/test/assets/expected-failure.js
new file mode 100644
index 0000000000..6d9555f11e
--- /dev/null
+++ b/test-runner/test/assets/expected-failure.js
@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../lib');
+
+it.fail(true)('fails',() => {
+  expect(1 + 1).toBe(3);
+});
+
+it('non-empty remaining',() => {
+  expect(1 + 1).toBe(2);
+});
diff --git a/test-runner/test/assets/nested-skip.js b/test-runner/test/assets/nested-skip.js
new file mode 100644
index 0000000000..2644394274
--- /dev/null
+++ b/test-runner/test/assets/nested-skip.js
@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../');
+
+describe.skip(true)('skipped', () => {
+  it('succeeds',() => {
+    expect(1 + 1).toBe(2);
+  });  
+});
diff --git a/test-runner/test/assets/one-timeout.js b/test-runner/test/assets/one-timeout.js
new file mode 100644
index 0000000000..2b0f4cebe5
--- /dev/null
+++ b/test-runner/test/assets/one-timeout.js
@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../');
+
+it('timeout', async () => {
+  await new Promise(f => setTimeout(f, 10000));
+});
diff --git a/test-runner/test/assets/unexpected-pass.js b/test-runner/test/assets/unexpected-pass.js
new file mode 100644
index 0000000000..114e650b1d
--- /dev/null
+++ b/test-runner/test/assets/unexpected-pass.js
@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+require('../../');
+
+it.fail(true)('succeeds',() => {
+  expect(1 + 1).toBe(2);
+});
diff --git a/test-runner/test/exit-code.spec.ts b/test-runner/test/exit-code.spec.ts
index bec6409fed..c9f557b544 100644
--- a/test-runner/test/exit-code.spec.ts
+++ b/test-runner/test/exit-code.spec.ts
@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import colors from 'colors/safe';
 import { spawnSync } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -30,6 +31,14 @@ it('should fail', async () => {
   expect(result.failed).toBe(1);
 });
 
+it('should timeout', async () => {
+  const { exitCode, passed, failed, timedOut } = await runTest('one-timeout.js', { timeout: 100 });
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(0);
+  expect(failed).toBe(0);
+  expect(timedOut).toBe(1);
+});
+
 it('should succeed', async () => {
   const result = await runTest('one-success.js');
   expect(result.exitCode).toBe(0);
@@ -85,6 +94,15 @@ it('should retry failures', async () => {
   expect(result.flaky).toBe(1);
 });
 
+it('should retry timeout', async () => {
+  const { exitCode, passed, failed, timedOut, output } = await runTest('one-timeout.js', { timeout: 100, retries: 2 });
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(0);
+  expect(failed).toBe(0);
+  expect(timedOut).toBe(1);
+  expect(output.split('\n')[0]).toBe(colors.red('T').repeat(3));
+});
+
 it('should repeat each', async () => {
   const { exitCode, report } = await runTest('one-success.js', { 'repeat-each': 3 });
   expect(exitCode).toBe(0);
@@ -106,6 +124,44 @@ it('should allow flaky', async () => {
   expect(result.flaky).toBe(1);
 });
 
+it('should fail on unexpected pass', async () => {
+  const { exitCode, failed, output } = await runTest('unexpected-pass.js');
+  expect(exitCode).toBe(1);
+  expect(failed).toBe(1);
+  expect(output).toContain('passed unexpectedly');
+});
+
+it('should fail on unexpected pass with retries', async () => {
+  const { exitCode, failed, output } = await runTest('unexpected-pass.js', { retries: 1 });
+  expect(exitCode).toBe(1);
+  expect(failed).toBe(1);
+  expect(output).toContain('passed unexpectedly');
+});
+
+it('should not retry unexpected pass', async () => {
+  const { exitCode, passed, failed, output } = await runTest('unexpected-pass.js', { retries: 2 });
+  expect(exitCode).toBe(1);
+  expect(passed).toBe(0);
+  expect(failed).toBe(1);
+  expect(output.split('\n')[0]).toBe(colors.red('P'));
+});
+
+it('should not retry expected failure', async () => {
+  const { exitCode, passed, failed, output } = await runTest('expected-failure.js', { retries: 2 });
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(2);
+  expect(failed).toBe(0);
+  expect(output.split('\n')[0]).toBe(colors.green('f') + colors.green('·'));
+});
+
+it('should respect nested skip', async () => {
+  const { exitCode, passed, failed, skipped } = await runTest('nested-skip.js');
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(0);
+  expect(failed).toBe(0);
+  expect(skipped).toBe(1);
+});
+
 async function runTest(filePath: string, params: any = {}) {
   const outputDir = path.join(__dirname, 'test-results');
   const reportFile = path.join(outputDir, 'results.json');
@@ -125,14 +181,20 @@ async function runTest(filePath: string, params: any = {}) {
   });
   const passed = (/(\d+) passed/.exec(output.toString()) || [])[1];
   const failed = (/(\d+) failed/.exec(output.toString()) || [])[1];
+  const timedOut = (/(\d+) timed out/.exec(output.toString()) || [])[1];
   const flaky = (/(\d+) flaky/.exec(output.toString()) || [])[1];
+  const skipped = (/(\d+) skipped/.exec(output.toString()) || [])[1];
   const report = JSON.parse(fs.readFileSync(reportFile).toString());
+  let outputStr = output.toString();
+  outputStr = outputStr.substring(1, outputStr.length - 1);
   return {
     exitCode: status,
-    output: output.toString(),
+    output: outputStr,
     passed: parseInt(passed, 10),
     failed: parseInt(failed || '0', 10),
+    timedOut: parseInt(timedOut || '0', 10),
     flaky: parseInt(flaky || '0', 10),
+    skipped: parseInt(skipped || '0', 10),
     report
   };
 }
diff --git a/test/__snapshots__/page-screenshot/firefox/screenshot-webgl.png b/test/__snapshots__/page-screenshot/firefox/screenshot-webgl.png
new file mode 100644
index 0000000000..76109512b5
Binary files /dev/null and b/test/__snapshots__/page-screenshot/firefox/screenshot-webgl.png differ
diff --git a/test/__snapshots__/page-screenshot/webkit/screenshot-webgl.png b/test/__snapshots__/page-screenshot/webkit/screenshot-webgl.png
new file mode 100644
index 0000000000..18d35c5ac1
Binary files /dev/null and b/test/__snapshots__/page-screenshot/webkit/screenshot-webgl.png differ
diff --git a/test/browsercontext-credentials.spec.ts b/test/browsercontext-credentials.spec.ts
index f535950127..a3a8341209 100644
--- a/test/browsercontext-credentials.spec.ts
+++ b/test/browsercontext-credentials.spec.ts
@@ -48,7 +48,7 @@ it('should work with correct credentials', async({browser, server}) => {
   await context.close();
 });
 
-it.fail(options.CHROMIUM && !options.HEADLESS)('should fail with wrong credentials', async({browser, server}) => {
+it('should fail with wrong credentials', async({browser, server}) => {
   server.setAuth('/empty.html', 'user', 'pass');
   const context = await browser.newContext({
     httpCredentials: { username: 'foo', password: 'bar' }
diff --git a/test/browsercontext-locale.spec.ts b/test/browsercontext-locale.spec.ts
index 14e5b80fdb..e7974c5a76 100644
--- a/test/browsercontext-locale.spec.ts
+++ b/test/browsercontext-locale.spec.ts
@@ -137,7 +137,7 @@ it('should be isolated between contexts', async({browser, server}) => {
   ]);
 });
 
-it.fail(options.FIREFOX)('should not change default locale in another context', async({browser, server}) => {
+it('should not change default locale in another context', async({browser, server}) => {
   async function getContextLocale(context) {
     const page = await context.newPage();
     return await page.evaluate(() => (new Intl.NumberFormat()).resolvedOptions().locale);
diff --git a/test/browsercontext-page-event.spec.ts b/test/browsercontext-page-event.spec.ts
index de62560d50..e8aa8c7657 100644
--- a/test/browsercontext-page-event.spec.ts
+++ b/test/browsercontext-page-event.spec.ts
@@ -156,7 +156,7 @@ it('should fire page lifecycle events', async function({browser, server}) {
   await context.close();
 });
 
-it.fail(options.WEBKIT)('should work with Shift-clicking', async({browser, server}) => {
+it.fixme(options.WEBKIT)('should work with Shift-clicking', async({browser, server}) => {
   // WebKit: Shift+Click does not open a new window.
   const context = await browser.newContext();
   const page = await context.newPage();
@@ -170,7 +170,7 @@ it.fail(options.WEBKIT)('should work with Shift-clicking', async({browser, serve
   await context.close();
 });
 
-it.fail(options.WEBKIT || options.FIREFOX)('should work with Ctrl-clicking', async({browser, server}) => {
+it.fixme(options.WEBKIT || options.FIREFOX)('should work with Ctrl-clicking', async({browser, server}) => {
   // Firefox: reports an opener in this case.
   // WebKit: Ctrl+Click does not open a new tab.
   const context = await browser.newContext();
diff --git a/test/browsercontext-timezone-id.spec.ts b/test/browsercontext-timezone-id.spec.ts
index 91d88f41dc..7bc4c71b8b 100644
--- a/test/browsercontext-timezone-id.spec.ts
+++ b/test/browsercontext-timezone-id.spec.ts
@@ -69,7 +69,7 @@ it('should work for multiple pages sharing same process', async({browser, server
   await context.close();
 });
 
-it.fail(options.FIREFOX)('should not change default timezone in another context', async({browser, server}) => {
+it('should not change default timezone in another context', async({browser, server}) => {
   async function getContextTimezone(context) {
     const page = await context.newPage();
     return await page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone);
diff --git a/test/browsertype-connect.spec.ts b/test/browsertype-connect.spec.ts
index 635bc2694d..8bd117dc83 100644
--- a/test/browsertype-connect.spec.ts
+++ b/test/browsertype-connect.spec.ts
@@ -81,7 +81,7 @@ it.skip(options.WIRE).slow()('disconnected event should be emitted when browser
   expect(disconnected2).toBe(1);
 });
 
-it.skip(options.WIRE).fail(options.CHROMIUM && WIN).slow()('should handle exceptions during connect', async({browserType, remoteServer}) => {
+it.skip(options.WIRE).slow()('should handle exceptions during connect', async({browserType, remoteServer}) => {
   const __testHookBeforeCreateBrowser = () => { throw new Error('Dummy') };
   const error = await browserType.connect({ wsEndpoint: remoteServer.wsEndpoint(), __testHookBeforeCreateBrowser } as any).catch(e => e);
   expect(error.message).toContain('Dummy');
diff --git a/test/browsertype-launch.spec.ts b/test/browsertype-launch.spec.ts
index 3e61877418..6d69b46aae 100644
--- a/test/browsertype-launch.spec.ts
+++ b/test/browsertype-launch.spec.ts
@@ -43,7 +43,7 @@ it.skip(options.FIREFOX)('should throw if page argument is passed', async({brows
   expect(waitError.message).toContain('can not specify page');
 });
 
-it.fail(true)('should reject if launched browser fails immediately', async({browserType, defaultBrowserOptions}) => {
+it.fixme(true)('should reject if launched browser fails immediately', async({browserType, defaultBrowserOptions}) => {
   // I'm getting ENCONRESET on this one.
   const options = Object.assign({}, defaultBrowserOptions, {executablePath: path.join(__dirname, 'assets', 'dummy_bad_browser_executable.js')});
   let waitError = null;
diff --git a/test/capabilities.spec.ts b/test/capabilities.spec.ts
index a067c99b92..4a631d41b0 100644
--- a/test/capabilities.spec.ts
+++ b/test/capabilities.spec.ts
@@ -48,7 +48,7 @@ it('should respect CSP', async({page, server}) => {
   expect(await page.evaluate(() => window['testStatus'])).toBe('SUCCESS');
 });
 
-it.fail(options.WEBKIT && (WIN || LINUX))('should play video', async({page, asset}) => {
+it.fixme(options.WEBKIT && (WIN || LINUX))('should play video', async({page, asset}) => {
   // TODO: the test passes on Windows locally but fails on GitHub Action bot,
   // apparently due to a Media Pack issue in the Windows Server.
   // Also the test is very flaky on Linux WebKit.
diff --git a/test/chromium/oopif.spec.ts b/test/chromium/oopif.spec.ts
index b48be9e835..5d3467ef50 100644
--- a/test/chromium/oopif.spec.ts
+++ b/test/chromium/oopif.spec.ts
@@ -65,7 +65,7 @@ it.skip(!options.CHROMIUM)('should handle remote -> local -> remote transitions'
   expect(await countOOPIFs(browser)).toBe(1);
 });
 
-it.fail(true)('should get the proper viewport', async({browser, page, server}) => {
+it.fixme(options.CHROMIUM).skip(!options.CHROMIUM)('should get the proper viewport', async({browser, page, server}) => {
   expect(page.viewportSize()).toEqual({width: 1280, height: 720});
   await page.goto(server.PREFIX + '/dynamic-oopif.html');
   expect(page.frames().length).toBe(2);
diff --git a/test/click-react.spec.ts b/test/click-react.spec.ts
index 4acba5a8e8..c1b345dce4 100644
--- a/test/click-react.spec.ts
+++ b/test/click-react.spec.ts
@@ -39,7 +39,7 @@ it.fail(true)('should report that selector does not match anymore', async ({page
   expect(error.message).toContain('element does not match the selector anymore');
 });
 
-it.fail(true)('should not retarget the handle when element is recycled', async ({page, server}) => {
+it.fixme(true)('should not retarget the handle when element is recycled', async ({page, server}) => {
   await page.goto(server.PREFIX + '/react.html');
   await page.evaluate(() => {
     renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2', disabled: true })] ));
@@ -66,7 +66,7 @@ it('should timeout when click opens alert', async({page, server}) => {
   await dialog.dismiss();
 });
 
-it.fail(true)('should retarget when element is recycled during hit testing', async ({page, server}) => {
+it.fixme(true)('should retarget when element is recycled during hit testing', async ({page, server}) => {
   await page.goto(server.PREFIX + '/react.html');
   await page.evaluate(() => {
     renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2' })] ));
@@ -81,7 +81,7 @@ it.fail(true)('should retarget when element is recycled during hit testing', asy
   expect(await page.evaluate('window.button2')).toBe(undefined);
 });
 
-it.fail(true)('should retarget when element is recycled before enabled check', async ({page, server}) => {
+it.fixme(true)('should retarget when element is recycled before enabled check', async ({page, server}) => {
   await page.goto(server.PREFIX + '/react.html');
   await page.evaluate(() => {
     renderComponent(e('div', {}, [e(MyButton, { name: 'button1' }), e(MyButton, { name: 'button2', disabled: true })] ));
diff --git a/test/click.spec.ts b/test/click.spec.ts
index 61b4cac874..26eed718c6 100644
--- a/test/click.spec.ts
+++ b/test/click.spec.ts
@@ -322,7 +322,7 @@ it('should click the button inside an iframe', async({page, server}) => {
   expect(await frame.evaluate(() => window['result'])).toBe('Clicked');
 });
 
-it.fail(options.CHROMIUM || options.WEBKIT)('should click the button with fixed position inside an iframe', async({page, server}) => {
+it.fixme(options.CHROMIUM || options.WEBKIT)('should click the button with fixed position inside an iframe', async({page, server}) => {
   // @see https://github.com/GoogleChrome/puppeteer/issues/4110
   // @see https://bugs.chromium.org/p/chromium/issues/detail?id=986390
   // @see https://chromium-review.googlesource.com/c/chromium/src/+/1742784
diff --git a/test/coverage.js b/test/coverage.js
index 18eb40f63e..86cb4e9cb3 100644
--- a/test/coverage.js
+++ b/test/coverage.js
@@ -24,12 +24,12 @@
 function traceAPICoverage(apiCoverage, api, events) {
   const uninstalls = [];
   for (const [name, classType] of Object.entries(api)) {
-    // console.log('trace', name);
     const className = name.substring(0, 1).toLowerCase() + name.substring(1);
     for (const methodName of Reflect.ownKeys(classType.prototype)) {
       const method = Reflect.get(classType.prototype, methodName);
       if (methodName === 'constructor' || typeof methodName !== 'string' || methodName.startsWith('_') || typeof method !== 'function')
         continue;
+ 
       apiCoverage.set(`${className}.${methodName}`, false);
       const override = function(...args) {
         apiCoverage.set(`${className}.${methodName}`, true);
diff --git a/test/defaultbrowsercontext-2.spec.ts b/test/defaultbrowsercontext-2.spec.ts
index b2636ed36b..a7d709536e 100644
--- a/test/defaultbrowsercontext-2.spec.ts
+++ b/test/defaultbrowsercontext-2.spec.ts
@@ -75,7 +75,7 @@ it('should support extraHTTPHeaders option', async ({server, launchPersistent})
   expect(request.headers['foo']).toBe('bar');
 });
 
-it('should accept userDataDir', async ({launchPersistent, tmpDir}) => {
+it.flaky(options.CHROMIUM)('should accept userDataDir', async ({launchPersistent, tmpDir}) => {
   const {page, context} = await launchPersistent();
   // Note: we need an open page to make sure its functional.
   expect(fs.readdirSync(tmpDir).length).toBeGreaterThan(0);
@@ -111,7 +111,7 @@ it.slow()('should restore state from userDataDir', async({browserType, defaultBr
   await removeUserDataDir(userDataDir2);
 });
 
-it.fail(options.CHROMIUM && (WIN || MAC)).slow()('should restore cookies from userDataDir', async({browserType, defaultBrowserOptions,  server, launchPersistent}) => {
+it.slow()('should restore cookies from userDataDir', async({browserType, defaultBrowserOptions,  server, launchPersistent}) => {
   const userDataDir = await makeUserDataDir();
   const browserContext = await browserType.launchPersistentContext(userDataDir, defaultBrowserOptions);
   const page = await browserContext.newPage();
diff --git a/test/dialog.spec.ts b/test/dialog.spec.ts
index 1bd6ece5c2..d5e18e2afd 100644
--- a/test/dialog.spec.ts
+++ b/test/dialog.spec.ts
@@ -62,7 +62,7 @@ it('should dismiss the confirm prompt', async({page}) => {
   expect(result).toBe(false);
 });
 
-it.fail(options.WEBKIT)('should be able to close context with open alert', async({browser}) => {
+it.fixme(options.WEBKIT && MAC)('should be able to close context with open alert', async({browser}) => {
   const context = await browser.newContext();
   const page = await context.newPage();
   const alertPromise = page.waitForEvent('dialog');
diff --git a/test/download.spec.ts b/test/download.spec.ts
index 332c48728b..4798c79c9f 100644
--- a/test/download.spec.ts
+++ b/test/download.spec.ts
@@ -250,7 +250,7 @@ it(`should report download path within page.on('download', …) handler for Blob
   expect(fs.readFileSync(path).toString()).toBe('Hello world');
   await page.close();
 })
-it.fail(options.FIREFOX || options.WEBKIT)('should report alt-click downloads', async({browser, server}) => {
+it.fixme(options.FIREFOX || options.WEBKIT)('should report alt-click downloads', async({browser, server}) => {
   // Firefox does not download on alt-click by default.
   // Our WebKit embedder does not download on alt-click, although Safari does.
   server.setRoute('/download', (req, res) => {
@@ -271,7 +271,7 @@ it.fail(options.FIREFOX || options.WEBKIT)('should report alt-click downloads',
   await page.close();
 });
 
-it.fail(options.CHROMIUM && !options.HEADLESS)('should report new window downloads', async({browser, server}) => {
+it.fixme(options.CHROMIUM && !options.HEADLESS)('should report new window downloads', async({browser, server}) => {
   // TODO: - the test fails in headful Chromium as the popup page gets closed along
   // with the session before download completed event arrives.
   // - WebKit doesn't close the popup page
diff --git a/test/elementhandle-owner-frame.spec.ts b/test/elementhandle-owner-frame.spec.ts
index d81352eff0..7d4ae67148 100644
--- a/test/elementhandle-owner-frame.spec.ts
+++ b/test/elementhandle-owner-frame.spec.ts
@@ -17,6 +17,7 @@
 
 import './playwright.fixtures';
 import utils from './utils';
+import { options } from './playwright.fixtures';
 
 it('should work', async ({ page, server }) => {
   await page.goto(server.EMPTY_PAGE);
@@ -34,7 +35,7 @@ it('should work for cross-process iframes', async ({ page, server }) => {
   expect(await elementHandle.ownerFrame()).toBe(frame);
 });
 
-it('should work for document', async ({ page, server }) => {
+it.flaky(WIN && options.WEBKIT)('should work for document', async ({ page, server }) => {
   await page.goto(server.EMPTY_PAGE);
   await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE);
   const frame = page.frames()[1];
diff --git a/test/elementhandle-screenshot.spec.ts b/test/elementhandle-screenshot.spec.ts
index 65d3861c4b..90659839e5 100644
--- a/test/elementhandle-screenshot.spec.ts
+++ b/test/elementhandle-screenshot.spec.ts
@@ -351,7 +351,7 @@ it.skip(ffheadful || options.WIRE)('should restore viewport after element screen
   await context.close();
 });
 
-it.skip(ffheadful)('should wait for element to stop moving', async({page, server, golden}) => {
+it.skip(ffheadful).flaky(options.WEBKIT && !options.HEADLESS && LINUX)('should wait for element to stop moving', async ({ page, server, golden }) => {
   await page.setViewportSize({ width: 500, height: 500 });
   await page.goto(server.PREFIX + '/grid.html');
   const elementHandle = await page.$('.box:nth-of-type(3)');
diff --git a/test/elementhandle-wait-for-element-state.spec.ts b/test/elementhandle-wait-for-element-state.spec.ts
index 440ecbdf8b..1c860f8493 100644
--- a/test/elementhandle-wait-for-element-state.spec.ts
+++ b/test/elementhandle-wait-for-element-state.spec.ts
@@ -16,6 +16,7 @@
  */
 
 import './playwright.fixtures';
+import { options } from './playwright.fixtures';
 
 async function giveItAChanceToResolve(page) {
   for (let i = 0; i < 5; i++)
diff --git a/test/frame-evaluate.spec.ts b/test/frame-evaluate.spec.ts
index 5d9e066401..2ca0423bbb 100644
--- a/test/frame-evaluate.spec.ts
+++ b/test/frame-evaluate.spec.ts
@@ -147,7 +147,7 @@ it.fail(options.CHROMIUM || options.FIREFOX)('should work in iframes that failed
   expect(await page.frames()[1].$('div')).toBeTruthy();
 });
 
-it.fail(options.CHROMIUM)('should work in iframes that interrupted initial javascript url navigation', async({page, server}) => {
+it.fixme(options.CHROMIUM)('should work in iframes that interrupted initial javascript url navigation', async({page, server}) => {
   // Chromium does not report isolated world for the iframe.
   await page.goto(server.EMPTY_PAGE);
   await page.evaluate(() => {
diff --git a/test/frame-hierarchy.spec.ts b/test/frame-hierarchy.spec.ts
index ec36d239d4..29ba88eda6 100644
--- a/test/frame-hierarchy.spec.ts
+++ b/test/frame-hierarchy.spec.ts
@@ -171,7 +171,7 @@ it('should report different frame instance when frame re-attaches', async({page,
   expect(frame1).not.toBe(frame2);
 });
 
-it.fail(options.FIREFOX)('should refuse to display x-frame-options:deny iframe', async({page, server}) => {
+it.fixme(options.FIREFOX)('should refuse to display x-frame-options:deny iframe', async({page, server}) => {
   server.setRoute('/x-frame-options-deny.html', async (req, res) => {
     res.setHeader('Content-Type', 'text/html');
     res.setHeader('X-Frame-Options', 'DENY');
diff --git a/test/headful.spec.ts b/test/headful.spec.ts
index 0a3221b2f8..39ecc6f2ec 100644
--- a/test/headful.spec.ts
+++ b/test/headful.spec.ts
@@ -128,7 +128,7 @@ it('should(not) block third party cookies', async({browserType, defaultBrowserOp
   await browser.close();
 });
 
-it.fail(options.WEBKIT)('should not override viewport size when passed null', async function({browserType, defaultBrowserOptions, server}) {
+it.fixme(options.WEBKIT)('should not override viewport size when passed null', async function({browserType, defaultBrowserOptions, server}) {
   // Our WebKit embedder does not respect window features.
   const browser = await browserType.launch({...defaultBrowserOptions, headless: false });
   const context = await browser.newContext({ viewport: null });
diff --git a/test/mouse.spec.ts b/test/mouse.spec.ts
index 565e983c7a..e242054f73 100644
--- a/test/mouse.spec.ts
+++ b/test/mouse.spec.ts
@@ -27,7 +27,7 @@ function dimensions() {
   };
 }
 
-it.fail(options.FIREFOX && WIN)('should click the document', async({page, server}) => {
+it.flaky(options.FIREFOX && WIN)('should click the document', async({page, server}) => {
   // Occasionally times out on options.FIREFOX on Windows: https://github.com/microsoft/playwright/pull/1911/checks?check_run_id=607149016
   await page.evaluate(() => {
     window["clickPromise"] = new Promise(resolve => {
diff --git a/test/page-add-script-tag.spec.ts b/test/page-add-script-tag.spec.ts
index 6204867d19..99636444f1 100644
--- a/test/page-add-script-tag.spec.ts
+++ b/test/page-add-script-tag.spec.ts
@@ -88,7 +88,7 @@ it('should work with content', async({page, server}) => {
   expect(await page.evaluate(() => window['__injected'])).toBe(35);
 });
 
-it.fail(options.FIREFOX)('should throw when added with content to the CSP page', async({page, server}) => {
+it('should throw when added with content to the CSP page', async({page, server}) => {
   // Firefox fires onload for blocked script before it issues the CSP console error.
   await page.goto(server.PREFIX + '/csp.html');
   let error = null;
diff --git a/test/page-emulate-media.spec.ts b/test/page-emulate-media.spec.ts
index fcb4d9cfdf..0a4660727a 100644
--- a/test/page-emulate-media.spec.ts
+++ b/test/page-emulate-media.spec.ts
@@ -114,7 +114,7 @@ it('should work in cross-process iframe', async({browser, server}) => {
   await page.close();
 });
 
-it.fail(options.FIREFOX)('should change the actual colors in css', async({page}) => {
+it('should change the actual colors in css', async({page}) => {
   await page.setContent(`
     <style>
       @media (prefers-color-scheme: dark) {
diff --git a/test/page-evaluate.spec.ts b/test/page-evaluate.spec.ts
index edf7856389..bfed454599 100644
--- a/test/page-evaluate.spec.ts
+++ b/test/page-evaluate.spec.ts
@@ -415,7 +415,7 @@ it('should not throw an error when evaluation does a navigation', async ({ page,
   expect(result).toEqual([42]);
 });
 
-it.fail(options.WEBKIT)('should not throw an error when evaluation does a synchronous navigation and returns an object', async ({ page, server }) => {
+it.fixme(options.WEBKIT)('should not throw an error when evaluation does a synchronous navigation and returns an object', async ({ page, server }) => {
   // It is imporant to be on about:blank for sync reload.
   const result = await page.evaluate(() => {
     window.location.reload();
@@ -433,7 +433,7 @@ it('should not throw an error when evaluation does a synchronous navigation and
   expect(result).toBe(undefined);
 });
 
-it.fail(options.WIRE)('should transfer 100Mb of data from page to node.js', async ({ page }) => {
+it('should transfer 100Mb of data from page to node.js', async ({ page }) => {
   // This is too slow with wire.
   const a = await page.evaluate(() => Array(100 * 1024 * 1024 + 1).join('a'));
   expect(a.length).toBe(100 * 1024 * 1024);
@@ -453,7 +453,7 @@ it('should work even when JSON is set to null', async ({ page }) => {
   expect(result).toEqual({ abc: 123 });
 });
 
-it.fail(options.FIREFOX)('should await promise from popup', async ({ page, server }) => {
+it('should await promise from popup', async ({ page, server }) => {
   // Something is wrong about the way Firefox waits for the chained promise
   await page.goto(server.EMPTY_PAGE);
   const result = await page.evaluate(() => {
diff --git a/test/page-event-crash.spec.ts b/test/page-event-crash.spec.ts
index 59b8f45ace..6f9a7dc1af 100644
--- a/test/page-event-crash.spec.ts
+++ b/test/page-event-crash.spec.ts
@@ -17,8 +17,6 @@
 
 import { options } from './playwright.fixtures';
 
-const CRASH_FAIL = (options.FIREFOX && WIN) || options.WIRE;
-// Firefox Win: it just doesn't crash sometimes.
 function crash(pageImpl, browserName) {
   if (browserName === 'chromium')
     pageImpl.mainFrame().goto('chrome://crash').catch(e => {});
@@ -28,13 +26,13 @@ function crash(pageImpl, browserName) {
     pageImpl._delegate._session.send('Page.crash', {}).catch(e => {});
 }
 
-it.fail(CRASH_FAIL)('should emit crash event when page crashes', async({page, browserName, toImpl}) => {
+it.fail(options.WIRE)('should emit crash event when page crashes', async({page, browserName, toImpl}) => {
   await page.setContent(`<div>This page should crash</div>`);
   crash(toImpl(page), browserName);
   await new Promise(f => page.on('crash', f));
 });
 
-it.fail(CRASH_FAIL)('should throw on any action after page crashes', async({page, browserName, toImpl}) => {
+it.fail(options.WIRE)('should throw on any action after page crashes', async({page, browserName, toImpl}) => {
   await page.setContent(`<div>This page should crash</div>`);
   crash(toImpl(page), browserName);
   await page.waitForEvent('crash');
@@ -43,7 +41,7 @@ it.fail(CRASH_FAIL)('should throw on any action after page crashes', async({page
   expect(err.message).toContain('crash');
 });
 
-it.fail(CRASH_FAIL)('should cancel waitForEvent when page crashes', async({page, browserName, toImpl}) => {
+it.fail(options.WIRE)('should cancel waitForEvent when page crashes', async({page, browserName, toImpl}) => {
   await page.setContent(`<div>This page should crash</div>`);
   const promise = page.waitForEvent('response').catch(e => e);
   crash(toImpl(page), browserName);
@@ -51,7 +49,7 @@ it.fail(CRASH_FAIL)('should cancel waitForEvent when page crashes', async({page,
   expect(error.message).toContain('Page crashed');
 });
 
-it.fail(CRASH_FAIL)('should cancel navigation when page crashes', async({page, browserName, toImpl, server}) => {
+it.fixme(options.WIRE)('should cancel navigation when page crashes', async({page, browserName, toImpl, server}) => {
   await page.setContent(`<div>This page should crash</div>`);
   server.setRoute('/one-style.css', () => {});
   const promise = page.goto(server.PREFIX + '/one-style.html').catch(e => e);
@@ -61,7 +59,7 @@ it.fail(CRASH_FAIL)('should cancel navigation when page crashes', async({page, b
   expect(error.message).toContain('Navigation failed because page crashed');
 });
 
-it.fail(CRASH_FAIL)('should be able to close context when page crashes', async({page, browserName, toImpl}) => {
+it.fixme(options.WIRE)('should be able to close context when page crashes', async({page, browserName, toImpl}) => {
   await page.setContent(`<div>This page should crash</div>`);
   crash(toImpl(page), browserName);
   await page.waitForEvent('crash');
diff --git a/test/page-screenshot.spec.ts b/test/page-screenshot.spec.ts
index 9672923478..78441c082f 100644
--- a/test/page-screenshot.spec.ts
+++ b/test/page-screenshot.spec.ts
@@ -214,7 +214,7 @@ it.skip(ffheadful)('should work for translateZ', async({page, server, golden}) =
   expect(screenshot).toMatchImage(golden('screenshot-translateZ.png'));
 });
 
-it.fail(options.FIREFOX || options.WEBKIT)('should work for webgl', async({page, server, golden}) => {
+it.fixme(options.FIREFOX).flaky(options.WEBKIT && LINUX)('should work for webgl', async({page, server, golden}) => {
   await page.setViewportSize({width: 640, height: 480});
   await page.goto(server.PREFIX + '/screenshots/webgl.html');
   const screenshot = await page.screenshot();
diff --git a/test/page-wait-for-navigation.spec.ts b/test/page-wait-for-navigation.spec.ts
index ac48f4860e..1a5b54e491 100644
--- a/test/page-wait-for-navigation.spec.ts
+++ b/test/page-wait-for-navigation.spec.ts
@@ -141,7 +141,7 @@ it('should work with DOM history.back()/history.forward()', async({page, server}
   expect(page.url()).toBe(server.PREFIX + '/second.html');
 });
 
-it.fail(options.FIREFOX)('should work when subframe issues window.stop()', async({page, server}) => {
+it('should work when subframe issues window.stop()', async({page, server}) => {
   server.setRoute('/frames/style.css', (req, res) => {});
   const navigationPromise = page.goto(server.PREFIX + '/frames/one-frame.html');
   const frame = await new Promise<Frame>(f => page.once('frameattached', f));
diff --git a/test/permissions.spec.ts b/test/permissions.spec.ts
index f7948a6fc7..704ceacd1d 100644
--- a/test/permissions.spec.ts
+++ b/test/permissions.spec.ts
@@ -98,7 +98,7 @@ describe.skip(options.WEBKIT)('permissions', () => {
     expect(await getPermission(page, 'geolocation')).toBe('prompt');
   });
 
-  it.fail(options.WEBKIT || options.FIREFOX || (options.CHROMIUM && !options.HEADLESS))('should trigger permission onchange', async({page, server, context}) => {
+  it.fail(options.WEBKIT || (options.CHROMIUM && !options.HEADLESS))('should trigger permission onchange', async({page, server, context}) => {
     //TODO: flaky
     // - Linux: https://github.com/microsoft/playwright/pull/1790/checks?check_run_id=587327883
     // - Win: https://ci.appveyor.com/project/aslushnikov/playwright/builds/32402536
diff --git a/test/screencast.spec.ts b/test/screencast.spec.ts
index 570b9e392d..738abf6a29 100644
--- a/test/screencast.spec.ts
+++ b/test/screencast.spec.ts
@@ -173,7 +173,7 @@ class VideoPlayer {
   }
 }
 
-it.fail(options.CHROMIUM)('should capture static page', async({page, tmpDir, videoPlayer, toImpl}) => {
+it.fixme(options.CHROMIUM)('should capture static page', async({page, tmpDir, videoPlayer, toImpl}) => {
   if (!toImpl)
     return;
   const videoFile = path.join(tmpDir, 'v.webm');
@@ -199,7 +199,7 @@ it.fail(options.CHROMIUM)('should capture static page', async({page, tmpDir, vid
   expectAll(pixels, almostRed);
 });
 
-it.fail(options.CHROMIUM)('should capture navigation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
+it.fixme(options.CHROMIUM).flaky(options.WEBKIT)('should capture navigation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
   if (!toImpl)
     return;
   const videoFile = path.join(tmpDir, 'v.webm');
@@ -233,7 +233,7 @@ it.fail(options.CHROMIUM)('should capture navigation', async({page, tmpDir, serv
 });
 
 // Accelerated compositing is disabled in WebKit on Windows.
-it.fail(options.CHROMIUM || (options.WEBKIT && WIN))('should capture css transformation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
+it.fixme(options.CHROMIUM || (options.WEBKIT && WIN)).flaky(options.WEBKIT && LINUX)('should capture css transformation', async({page, tmpDir, server, videoPlayer, toImpl}) => {
   if (!toImpl)
     return;
   const videoFile = path.join(tmpDir, 'v.webm');
@@ -258,7 +258,7 @@ it.fail(options.CHROMIUM || (options.WEBKIT && WIN))('should capture css transfo
   }
 });
 
-it.fail(options.CHROMIUM)('should fire start/stop events when page created/closed', async({browser, tmpDir, server, toImpl}) => {
+it.slow().fixme(options.CHROMIUM)('should fire start/stop events when page created/closed', async({browser, tmpDir, server, toImpl}) => {
   if (!toImpl)
    return;
   // Use server side of the context. All the code below also uses server side APIs.
@@ -280,7 +280,7 @@ it.fail(options.CHROMIUM)('should fire start/stop events when page created/close
   await context.close();
 });
 
-it.fail(options.CHROMIUM)('should fire start event for popups', async({browser, tmpDir, server, toImpl}) => {
+it.fixme(options.CHROMIUM)('should fire start event for popups', async({browser, tmpDir, server, toImpl}) => {
   if (!toImpl)
    return;
   // Use server side of the context. All the code below also uses server side APIs.