From a12112c24d26ef5186830ea7b729732c0bda813d Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Wed, 14 Sep 2022 15:05:18 -0700 Subject: [PATCH] devops(docker): add docker integration smoke tests (#17267) --- .github/workflows/tests_primary.yml | 1 + .github/workflows/tests_secondary.yml | 27 +++++++++++++++++++++++++++ package.json | 2 +- tests/config/platformFixtures.ts | 2 +- tests/config/testMode.ts | 2 +- tests/config/testModeFixtures.ts | 1 + tests/library/capabilities.spec.ts | 3 ++- tests/library/download.spec.ts | 3 ++- tests/library/logger.spec.ts | 3 ++- tests/library/playwright.config.ts | 12 +++++++++--- tests/library/proxy.spec.ts | 3 ++- tests/page/page-event-network.spec.ts | 12 ++++++------ 12 files changed, 55 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index 5ac22dee08..f4a348f8ed 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -184,3 +184,4 @@ jobs: - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash + diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 136d763cfc..b798c07879 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -801,3 +801,30 @@ jobs: - run: npm run build - run: npx playwright install-deps - run: utils/build/build-playwright-driver.sh + + smoke_test_docker_integration: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 14 + - run: npm i -g npm@8 + - run: npm ci + env: + DEBUG: pw:install + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + - run: npm run build + - run: npx playwright install --with-deps + - run: ./utils/docker/build.sh --amd64 focal playwright:localbuild + - run: PWTEST_DOCKER_BASE_IMAGE=playwright:localbuild npx playwright docker build + - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run dtest + - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json + if: always() + shell: bash + - uses: actions/upload-artifact@v3 + if: always() + with: + name: docker-integration-test-results + path: test-results + diff --git a/package.json b/package.json index 58491ebb60..b44b25bc16 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ }, "license": "Apache-2.0", "scripts": { + "dtest": "cross-env PLAYWRIGHT_DOCKER=1 playwright docker test --config=tests/library/playwright.config.ts --grep '@smoke'", "ctest": "playwright test --config=tests/library/playwright.config.ts --project=chromium", "ftest": "playwright test --config=tests/library/playwright.config.ts --project=firefox", "wtest": "playwright test --config=tests/library/playwright.config.ts --project=webkit", @@ -24,7 +25,6 @@ "test-html-reporter": "playwright test --config=packages/html-reporter", "test-web": "playwright test --config=packages/web", "ttest": "node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright.config.ts", - "vtest": "cross-env PLAYWRIGHT_DOCKER=1 node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright.config.ts", "ct": "playwright test tests/components/test-all.spec.js --reporter=list", "test": "playwright test --config=tests/library/playwright.config.ts", "eslint": "eslint --ext ts,tsx .", diff --git a/tests/config/platformFixtures.ts b/tests/config/platformFixtures.ts index 4a044c8dcc..bd318803ea 100644 --- a/tests/config/platformFixtures.ts +++ b/tests/config/platformFixtures.ts @@ -24,7 +24,7 @@ export type PlatformWorkerFixtures = { }; export const platformTest = test.extend<{}, PlatformWorkerFixtures>({ - platform: [process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' }], + platform: [process.env.PLAYWRIGHT_DOCKER ? 'linux' : process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' }], isWindows: [process.platform === 'win32', { scope: 'worker' }], isMac: [process.platform === 'darwin', { scope: 'worker' }], isLinux: [process.platform === 'linux', { scope: 'worker' }], diff --git a/tests/config/testMode.ts b/tests/config/testMode.ts index 02623b6dce..a90aac2df3 100644 --- a/tests/config/testMode.ts +++ b/tests/config/testMode.ts @@ -17,7 +17,7 @@ import { start } from '../../packages/playwright-core/lib/outofprocess'; import type { Playwright } from '../../packages/playwright-core/lib/client/playwright'; -export type TestModeName = 'default' | 'driver' | 'service' | 'service2'; +export type TestModeName = 'default' | 'driver' | 'service' | 'service2' | 'docker'; interface TestMode { setup(): Promise; diff --git a/tests/config/testModeFixtures.ts b/tests/config/testModeFixtures.ts index 43d7974729..c18ae843fc 100644 --- a/tests/config/testModeFixtures.ts +++ b/tests/config/testModeFixtures.ts @@ -37,6 +37,7 @@ export const testModeTest = test.extend { const testMode = { default: new DefaultTestMode(), + docker: new DefaultTestMode(), service: new DefaultTestMode(), driver: new DriverTestMode(), service2: new DefaultTestMode(), diff --git a/tests/library/capabilities.spec.ts b/tests/library/capabilities.spec.ts index f57168b994..9e934cf460 100644 --- a/tests/library/capabilities.spec.ts +++ b/tests/library/capabilities.spec.ts @@ -65,7 +65,8 @@ it('should respect CSP @smoke', async ({ page, server }) => { expect(await page.evaluate(() => window['testStatus'])).toBe('SUCCESS'); }); -it('should play video @smoke', async ({ page, asset, browserName, platform }) => { +it('should play video @smoke', async ({ page, asset, browserName, platform, mode }) => { + it.skip(mode === 'docker', 'local paths do not work with remote setup'); // TODO: the test passes on Windows locally but fails on GitHub Action bot, // apparently due to a Media Pack issue in the Windows Server. // Also the test is very flaky on Linux WebKit. diff --git a/tests/library/download.spec.ts b/tests/library/download.spec.ts index 4df67752dc..f2683d4897 100644 --- a/tests/library/download.spec.ts +++ b/tests/library/download.spec.ts @@ -50,7 +50,8 @@ it.describe('download event', () => { }); }); - it('should report download when navigation turns into download @smoke', async ({ browser, server, browserName }) => { + it('should report download when navigation turns into download @smoke', async ({ browser, server, browserName, mode }) => { + it.skip(mode === 'docker', 'local paths do not work remote connection'); const page = await browser.newPage(); const [download, responseOrError] = await Promise.all([ page.waitForEvent('download'), diff --git a/tests/library/logger.spec.ts b/tests/library/logger.spec.ts index 7e3b73491c..8ee19245c3 100644 --- a/tests/library/logger.spec.ts +++ b/tests/library/logger.spec.ts @@ -16,7 +16,8 @@ import { playwrightTest as it, expect } from '../config/browserTest'; -it('should log @smoke', async ({ browserType }) => { +it('should log @smoke', async ({ browserType, mode }) => { + it.fixme(mode === 'docker', 'logger is not plumbed into the remote connection'); const log = []; const browser = await browserType.launch({ logger: { log: (name, severity, message) => log.push({ name, severity, message }), diff --git a/tests/library/playwright.config.ts b/tests/library/playwright.config.ts index 17c49b2e35..7829d55e7e 100644 --- a/tests/library/playwright.config.ts +++ b/tests/library/playwright.config.ts @@ -20,6 +20,7 @@ loadEnv({ path: path.join(__dirname, '..', '..', '.env') }); import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions } from '@playwright/test'; import * as path from 'path'; import type { TestModeWorkerOptions } from '../config/testModeFixtures'; +import type { TestModeName } from '../config/testMode'; import type { CoverageWorkerOptions } from '../config/coverageFixtures'; type BrowserName = 'chromium' | 'firefox' | 'webkit'; @@ -33,9 +34,13 @@ const getExecutablePath = (browserName: BrowserName) => { return process.env.WKPATH; }; -const mode = process.env.PW_OUT_OF_PROCESS_DRIVER ? - 'driver' : - (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service' | 'service2'); +let mode: TestModeName = 'default'; +if (process.env.PW_OUT_OF_PROCESS_DRIVER) + mode = 'driver'; +else if (process.env.PLAYWRIGHT_DOCKER) + mode = 'docker'; +else + mode = (process.env.PWTEST_MODE ?? 'default') as ('default' | 'driver' | 'service' | 'service2'); const headed = process.argv.includes('--headed'); const channel = process.env.PWTEST_CHANNEL as any; const video = !!process.env.PWTEST_VIDEO; @@ -129,6 +134,7 @@ for (const browserName of browserNames) { metadata: { platform: process.platform, docker: !!process.env.INSIDE_DOCKER, + dockerIntegration: !!process.env.PLAYWRIGHT_DOCKER, headful: !!headed, browserName, channel, diff --git a/tests/library/proxy.spec.ts b/tests/library/proxy.spec.ts index c339756763..1d1d755cac 100644 --- a/tests/library/proxy.spec.ts +++ b/tests/library/proxy.spec.ts @@ -28,7 +28,8 @@ it('should throw for bad server value', async ({ browserType }) => { expect(error.message).toContain('proxy.server: expected string, got number'); }); -it('should use proxy @smoke', async ({ browserType, server }) => { +it('should use proxy @smoke', async ({ browserType, server, mode }) => { + it.skip(mode === 'docker', 'proxy is not supported for remote connection'); server.setRoute('/target.html', async (req, res) => { res.end('Served by the proxy'); }); diff --git a/tests/page/page-event-network.spec.ts b/tests/page/page-event-network.spec.ts index 3b5a9641d6..505ca3f673 100644 --- a/tests/page/page-event-network.spec.ts +++ b/tests/page/page-event-network.spec.ts @@ -42,7 +42,7 @@ it('Page.Events.Response @smoke', async ({ page, server }) => { expect(responses[0].request()).toBeTruthy(); }); -it('Page.Events.RequestFailed @smoke', async ({ page, server, browserName, isMac, isWindows }) => { +it('Page.Events.RequestFailed @smoke', async ({ page, server, browserName, platform }) => { server.setRoute('/one-style.css', (req, res) => { res.setHeader('Content-Type', 'text/css'); res.connection.destroy(); @@ -57,12 +57,12 @@ it('Page.Events.RequestFailed @smoke', async ({ page, server, browserName, isMac if (browserName === 'chromium') { expect(failedRequests[0].failure().errorText).toBe('net::ERR_EMPTY_RESPONSE'); } else if (browserName === 'webkit') { - if (isMac) - expect(failedRequests[0].failure().errorText).toBe('The network connection was lost.'); - else if (isWindows) - expect(failedRequests[0].failure().errorText).toBe('Server returned nothing (no headers, no data)'); - else + if (platform === 'linux') expect(failedRequests[0].failure().errorText).toBe('Message Corrupt'); + else if (platform === 'darwin') + expect(failedRequests[0].failure().errorText).toBe('The network connection was lost.'); + else if (platform === 'win32') + expect(failedRequests[0].failure().errorText).toBe('Server returned nothing (no headers, no data)'); } else { expect(failedRequests[0].failure().errorText).toBe('NS_ERROR_NET_RESET'); }