From 4aaf3b7596d9ca05b2fb7a6a62d5f9576b4f2660 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 23 Sep 2020 14:52:06 -0700 Subject: [PATCH] test: roll test runner to 0.3.20 (#3963) --- package-lock.json | 44 ++++++++++++++++---------- package.json | 2 +- test/browsertype-launch-server.spec.ts | 6 ++-- test/playwright.fixtures.ts | 32 +++++++++---------- test/proxy.spec.ts | 4 +-- test/screencast.spec.ts | 14 ++++---- 6 files changed, 57 insertions(+), 45 deletions(-) diff --git a/package-lock.json b/package-lock.json index df82f02b8e..d476d59fbf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1187,9 +1187,9 @@ } }, "@playwright/test-runner": { - "version": "0.3.18", - "resolved": "https://registry.npmjs.org/@playwright/test-runner/-/test-runner-0.3.18.tgz", - "integrity": "sha512-QvkStde6VE6pV0oukwV/U2P1aT96sVndbaMZAmXx2w31HotOl2w/G4tWVAIV/UtKtcI3fFY2womfUB3wUAAyjw==", + "version": "0.3.20", + "resolved": "https://registry.npmjs.org/@playwright/test-runner/-/test-runner-0.3.20.tgz", + "integrity": "sha512-G3gd5ELp2MesOq7xoFbhinnFARi5YyNiFdB3hnazAP3EjYpZUrzZBP7/BLLur6jzwiNmomq1NbEBn4SHiXUDiQ==", "dev": true, "requires": { "@babel/code-frame": "^7.10.4", @@ -1199,6 +1199,7 @@ "@babel/preset-typescript": "^7.10.4", "colors": "^1.4.0", "commander": "^6.1.0", + "debug": "^4.1.5", "expect": "^26.4.2", "jpeg-js": "^0.4.2", "micromatch": "^4.0.2", @@ -1211,6 +1212,17 @@ "stack-utils": "^2.0.2", "terminal-link": "^2.1.1", "text-diff": "^1.0.1" + }, + "dependencies": { + "debug": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", + "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + } } }, "@sindresorhus/is": { @@ -1378,9 +1390,9 @@ } }, "@types/yargs": { - "version": "15.0.5", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.5.tgz", - "integrity": "sha512-Dk/IDOPtOgubt/IaevIUbTgV7doaKkoorvOyYM2CMwuDyP89bekI7H4xLIwunNYiK9jhCkmc6pUrJk3cj2AB9w==", + "version": "15.0.7", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.7.tgz", + "integrity": "sha512-Gf4u3EjaPNcC9cTu4/j2oN14nSVhr8PQ+BvBcBQHAhDZfl0bVIiLgvnRXv/dn58XhTm9UXvBpvJpDlwV65QxOA==", "dev": true, "requires": { "@types/yargs-parser": "*" @@ -2183,12 +2195,12 @@ } }, "browserslist": { - "version": "4.14.3", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.14.3.tgz", - "integrity": "sha512-GcZPC5+YqyPO4SFnz48/B0YaCwS47Q9iPChRGi6t7HhflKBcINzFrJvRfC+jp30sRMKxF+d4EHGs27Z0XP1NaQ==", + "version": "4.14.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.14.4.tgz", + "integrity": "sha512-7FOuawafVdEwa5Jv4nzeik/PepAjVte6HmVGHsjt2bC237jeL9QlcTBDF3PnHEvcC6uHwLGYPwZHNZMB7wWAnw==", "dev": true, "requires": { - "caniuse-lite": "^1.0.30001131", + "caniuse-lite": "^1.0.30001135", "electron-to-chromium": "^1.3.570", "escalade": "^3.1.0", "node-releases": "^1.1.61" @@ -2321,9 +2333,9 @@ "dev": true }, "caniuse-lite": { - "version": "1.0.30001133", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001133.tgz", - "integrity": "sha512-s3XAUFaC/ntDb1O3lcw9K8MPeOW7KO3z9+GzAoBxfz1B0VdacXPMKgFUtG4KIsgmnbexmi013s9miVu4h+qMHw==", + "version": "1.0.30001135", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001135.tgz", + "integrity": "sha512-ziNcheTGTHlu9g34EVoHQdIu5g4foc8EsxMGC7Xkokmvw0dqNtX8BS8RgCgFBaAiSp2IdjvBxNdh0ssib28eVQ==", "dev": true }, "caseless": { @@ -3003,9 +3015,9 @@ } }, "electron-to-chromium": { - "version": "1.3.570", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.570.tgz", - "integrity": "sha512-Y6OCoVQgFQBP5py6A/06+yWxUZHDlNr/gNDGatjH8AZqXl8X0tE4LfjLJsXGz/JmWJz8a6K7bR1k+QzZ+k//fg==", + "version": "1.3.571", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.571.tgz", + "integrity": "sha512-UYEQ2Gtc50kqmyOmOVtj6Oqi38lm5yRJY3pLuWt6UIot0No1L09uu6Ja6/1XKwmz/p0eJFZTUZi+khd1PV1hHA==", "dev": true }, "elliptic": { diff --git a/package.json b/package.json index 27be14e47b..6c523ea54d 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "ws": "^7.3.1" }, "devDependencies": { - "@playwright/test-runner": "^0.3.18", + "@playwright/test-runner": "^0.3.20", "@types/debug": "^4.1.5", "@types/extract-zip": "^1.6.2", "@types/mime": "^2.0.3", diff --git a/test/browsertype-launch-server.spec.ts b/test/browsertype-launch-server.spec.ts index 38cde55ff7..b2c182882a 100644 --- a/test/browsertype-launch-server.spec.ts +++ b/test/browsertype-launch-server.spec.ts @@ -26,9 +26,9 @@ describe('lauch server', suite => { await browserServer.close(); }); - it('should work with port', async ({browserType, defaultBrowserOptions, workerIndex}) => { - const browserServer = await browserType.launchServer({ ...defaultBrowserOptions, port: 8800 + workerIndex }); - expect(browserServer.wsEndpoint()).toContain(String(8800 + workerIndex)); + it('should work with port', async ({browserType, defaultBrowserOptions, testWorkerIndex}) => { + const browserServer = await browserType.launchServer({ ...defaultBrowserOptions, port: 8800 + testWorkerIndex }); + expect(browserServer.wsEndpoint()).toContain(String(8800 + testWorkerIndex)); await browserServer.close(); }); diff --git a/test/playwright.fixtures.ts b/test/playwright.fixtures.ts index b23c109af0..f8a6323088 100644 --- a/test/playwright.fixtures.ts +++ b/test/playwright.fixtures.ts @@ -61,6 +61,7 @@ type PlaywrightTestFixtures = { httpsServer: TestServer; browserServer: BrowserServer; testOutputDir: string; + tmpDir: string; createUserDataDir: () => Promise; launchPersistent: (options?: Parameters['launchPersistentContext']>[1]) => Promise<{context: BrowserContext, page: Page}>; }; @@ -97,11 +98,11 @@ export const options = { TRACING: valueFromEnv('TRACING', false), }; -defineWorkerFixture('httpService', async ({workerIndex}, test) => { +defineWorkerFixture('httpService', async ({ testWorkerIndex }, test) => { const assetsPath = path.join(__dirname, 'assets'); const cachedPath = path.join(__dirname, 'assets', 'cached'); - const port = 8907 + workerIndex * 2; + const port = 8907 + testWorkerIndex * 2; const server = await TestServer.create(assetsPath, port); server.enableHTTPCache(cachedPath); @@ -126,7 +127,7 @@ const getExecutablePath = browserName => { return process.env.WKPATH; }; -defineWorkerFixture('defaultBrowserOptions', async ({browserName}, runTest, config) => { +defineWorkerFixture('defaultBrowserOptions', async ({ browserName, testConfig }, runTest) => { const executablePath = getExecutablePath(browserName); if (executablePath) console.error(`Using executable at ${executablePath}`); @@ -135,11 +136,11 @@ defineWorkerFixture('defaultBrowserOptions', async ({browserName}, runTest, conf slowMo: options.SLOW_MO, headless: options.HEADLESS, executablePath, - artifactsPath: config.outputDir, + artifactsPath: testConfig.outputDir, }); }); -defineWorkerFixture('playwright', async ({browserName, workerIndex, platform}, test) => { +defineWorkerFixture('playwright', async ({browserName, testWorkerIndex, platform}, test) => { assert(platform); // Depend on platform to generate all tests. const {coverage, uninstall} = installCoverageHooks(browserName); if (options.WIRE) { @@ -172,7 +173,7 @@ defineWorkerFixture('playwright', async ({browserName, workerIndex, platform}, t async function teardownCoverage() { uninstall(); - const coveragePath = path.join(__dirname, 'coverage-report', workerIndex + '.json'); + const coveragePath = path.join(__dirname, 'coverage-report', testWorkerIndex + '.json'); const coverageJSON = [...coverage.keys()].filter(key => coverage.get(key)); await fs.promises.mkdir(path.dirname(coveragePath), { recursive: true }); await fs.promises.writeFile(coveragePath, JSON.stringify(coverageJSON, undefined, 2), 'utf8'); @@ -259,10 +260,10 @@ defineWorkerFixture('expectedSSLError', async ({browserName, platform}, runTest) await runTest(expectedSSLError); }); -defineTestFixture('testOutputDir', async ({}, runTest, info) => { - const relativePath = path.relative(info.config.testDir, info.file).replace(/\.spec\.[jt]s/, ''); - const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_'); - const testOutputDir = path.join(info.config.outputDir, relativePath, sanitizedTitle); +defineTestFixture('testOutputDir', async ({ testInfo }, runTest) => { + const relativePath = path.relative(testInfo.config.testDir, testInfo.file).replace(/\.spec\.[jt]s/, ''); + const sanitizedTitle = testInfo.title.replace(/[^\w\d]+/g, '_'); + const testOutputDir = path.join(testInfo.config.outputDir, relativePath, sanitizedTitle); await fs.promises.mkdir(testOutputDir, { recursive: true }); await runTest(testOutputDir); const files = await fs.promises.readdir(testOutputDir); @@ -273,10 +274,9 @@ defineTestFixture('testOutputDir', async ({}, runTest, info) => { } }); -defineTestFixture('context', async ({browser, testOutputDir}, runTest, info) => { - const { config } = info; +defineTestFixture('context', async ({ browser, testOutputDir, testConfig }, runTest) => { const contextOptions: BrowserContextOptions = { - relativeArtifactsPath: path.relative(config.outputDir, testOutputDir), + relativeArtifactsPath: path.relative(testConfig.outputDir, testOutputDir), recordTrace: !!options.TRACING, // TODO: enable videos. Currently, long videos are slowly processed by Chromium // and (sometimes) Firefox, which causes test timeouts. @@ -287,14 +287,14 @@ defineTestFixture('context', async ({browser, testOutputDir}, runTest, info) => await context.close(); }); -defineTestFixture('page', async ({context, testOutputDir}, runTest, info) => { +defineTestFixture('page', async ({ context, testOutputDir, testInfo }, runTest) => { const page = await context.newPage(); await runTest(page); - if (info.status === 'failed' || info.status === 'timedOut') + if (testInfo.status === 'failed' || testInfo.status === 'timedOut') await page.screenshot({ timeout: 5000, path: path.join(testOutputDir, 'test-failed.png') }); }); -defineTestFixture('createUserDataDir', async ({testOutputDir}, runTest, info) => { +defineTestFixture('createUserDataDir', async ({testOutputDir}, runTest) => { let counter = 0; const dirs: string[] = []; async function createUserDataDir() { diff --git a/test/proxy.spec.ts b/test/proxy.spec.ts index f7d2d23b52..a7645f2d95 100644 --- a/test/proxy.spec.ts +++ b/test/proxy.spec.ts @@ -121,7 +121,7 @@ it('should exclude patterns', (test, parameters) => { it('should use socks proxy', (test, parameters) => { test.flaky(options.MAC(parameters) && options.WEBKIT(parameters), 'Intermittent page.goto: The network connection was lost error on bots'); -}, async ({ browserType, defaultBrowserOptions, workerIndex }) => { +}, async ({ browserType, defaultBrowserOptions, testWorkerIndex }) => { const server = socks.createServer((info, accept, deny) => { let socket; if ((socket = accept(true))) { @@ -138,7 +138,7 @@ it('should use socks proxy', (test, parameters) => { ].join('\r\n')); } }); - const socksPort = 9107 + workerIndex * 2; + const socksPort = 9107 + testWorkerIndex * 2; server.listen(socksPort, 'localhost'); server.useAuth(socks.auth.None()); diff --git a/test/screencast.spec.ts b/test/screencast.spec.ts index 81da6cf532..51150d722a 100644 --- a/test/screencast.spec.ts +++ b/test/screencast.spec.ts @@ -32,11 +32,11 @@ type TestState = { const fixtures = playwrightFixtures.declareWorkerFixtures().declareTestFixtures(); const { it, expect, describe, defineTestFixture, defineWorkerFixture, overrideWorkerFixture } = fixtures; -overrideWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test, config) => { +overrideWorkerFixture('browser', async ({ browserType, defaultBrowserOptions, testConfig }, test) => { const browser = await browserType.launch({ ...defaultBrowserOptions, // Make sure videos are stored on the same volume as the test output dir. - artifactsPath: path.join(config.outputDir, '.screencast'), + artifactsPath: path.join(testConfig.outputDir, '.screencast'), }); await test(browser); await browser.close(); @@ -57,14 +57,14 @@ defineTestFixture('videoPlayer', async ({videoPlayerBrowser, server}, test) => { await page.close(); }); -defineTestFixture('relativeArtifactsPath', async ({browserType}, runTest, info) => { - const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_'); +defineTestFixture('relativeArtifactsPath', async ({ browserType, testInfo }, runTest) => { + const sanitizedTitle = testInfo.title.replace(/[^\w\d]+/g, '_'); const relativeArtifactsPath = `${browserType.name()}-${sanitizedTitle}`; await runTest(relativeArtifactsPath); }); -defineTestFixture('videoDir', async ({relativeArtifactsPath}, runTest, info) => { - await runTest(path.join(info.config.outputDir, '.screencast', relativeArtifactsPath)); +defineTestFixture('videoDir', async ({ relativeArtifactsPath, testConfig }, runTest) => { + await runTest(path.join(testConfig.outputDir, '.screencast', relativeArtifactsPath)); }); function almostRed(r, g, b, alpha) { @@ -232,7 +232,7 @@ describe('screencast', suite => { expectAll(pixels, almostRed); }); - it('should capture navigation', (test, parameters) => { + it('should capture navigation', test => { test.flaky(); }, async ({browser, server, videoPlayer, relativeArtifactsPath, videoDir}) => { const context = await browser.newContext({