Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore: skip tests that fail in cross-machine service mode (#24412)

Browse source
This commit is contained in:
Pavel Feldman 2023-07-25 16:47:04 -07:00 committed by GitHub
parent ed14bf2103
commit b39fd7283f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 90 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/playwright-core/src/utils/userAgent.ts
View file

@ -77,5 +77,7 @@ export function getEmbedderName(): { embedderName: string, embedderVersion: stri
export function getPlaywrightVersion(majorMinorOnly = false): string {
const packageJson = require('./../../package.json');
if (process.env.PW_VERSION_OVERRIDE)
return process.env.PW_VERSION_OVERRIDE;
return majorMinorOnly ? packageJson.version.split('.').slice(0, 2).join('.') : packageJson.version;
}

9
tests/config/browserTest.ts
View file

@ -91,7 +91,8 @@ const test = baseTest.extend<BrowserTestTestFixtures, BrowserTestWorkerFixtures>
await run(_contextFactory);
},
createUserDataDir: async ({}, run) => {
createUserDataDir: async ({ mode }, run) => {
test.skip(mode.startsWith('service'));
const dirs: string[] = [];
// We do not put user data dir in testOutputPath,
// because we do not want to upload them as test result artifacts.
@ -107,7 +108,8 @@ const test = baseTest.extend<BrowserTestTestFixtures, BrowserTestWorkerFixtures>
await removeFolders(dirs);
},
launchPersistent: async ({ createUserDataDir, browserType }, run) => {
launchPersistent: async ({ createUserDataDir, browserType, mode }, run) => {
test.skip(mode.startsWith('service'));
let persistentContext: BrowserContext | undefined;
await run(async options => {
if (persistentContext)
@ -121,7 +123,8 @@ const test = baseTest.extend<BrowserTestTestFixtures, BrowserTestWorkerFixtures>
await persistentContext.close();
},
startRemoteServer: async ({ childProcess, browserType }, run) => {
startRemoteServer: async ({ childProcess, browserType, mode }, run) => {
test.skip(mode.startsWith('service'));
let server: PlaywrightServer | undefined;
const fn = async (kind: 'launchServer' | 'run-server', options?: RemoteServerOptions) => {
if (server)

2
tests/config/testMode.ts
View file

@ -17,7 +17,7 @@
import { start } from '../../packages/playwright-core/lib/outofprocess';
import type { Playwright } from '../../packages/playwright-core/lib/client/playwright';
export type TestModeName = 'default' | 'driver' | 'service';
export type TestModeName = 'default' | 'driver' | 'service' | 'service2';
interface TestMode {
setup(): Promise<Playwright>;

1
tests/config/testModeFixtures.ts
View file

@ -38,6 +38,7 @@ export const testModeTest = test.extend<TestModeTestFixtures, TestModeWorkerOpti
const testMode = {
default: new DefaultTestMode(),
service: new DefaultTestMode(),
service2: new DefaultTestMode(),
driver: new DriverTestMode(),
}[mode];
require('playwright-core/lib/utils').setUnderTest();

2
tests/library/browsercontext-proxy.spec.ts
View file

@ -16,6 +16,8 @@
import { browserTest as it, expect } from '../config/browserTest';
it.skip(({ mode }) => mode.startsWith('service'));
it.use({
launchOptions: async ({ launchOptions }, use) => {
await use({

3
tests/library/browsertype-basic.spec.ts
View file

@ -18,8 +18,9 @@
import fs from 'fs';
import { playwrightTest as test, expect } from '../config/browserTest';
test('browserType.executablePath should work', async ({ browserType, channel }) => {
test('browserType.executablePath should work', async ({ browserType, channel, mode }) => {
test.skip(!!channel, 'We skip browser download when testing a channel');
test.skip(mode.startsWith('service'));
test.skip(!!(browserType as any)._defaultLaunchOptions.executablePath, 'Skip with custom executable path');
const executablePath = browserType.executablePath();

4
tests/library/browsertype-connect.spec.ts
View file

@ -70,6 +70,7 @@ const test = playwrightTest.extend<ExtraFixtures>({
});
test.slow(true, 'All connect tests are slow');
test.skip(({ mode }) => mode.startsWith('service'));
for (const kind of ['launchServer', 'run-server'] as const) {
test.describe(kind, () => {
@ -644,7 +645,8 @@ for (const kind of ['launchServer', 'run-server'] as const) {
expect(await response.json()).toEqual({ 'foo': 'bar' });
});
test('should upload large file', async ({ connect, startRemoteServer, server, browserName, isMac }, testInfo) => {
test('should upload large file', async ({ connect, startRemoteServer, server, browserName, isMac, mode }, testInfo) => {
test.skip(mode.startsWith('service'), 'Take it easy on service');
test.skip(browserName === 'webkit' && isMac && parseInt(os.release(), 10) < 20, 'WebKit for macOS 10.15 is frozen and does not have corresponding protocol features.');
test.slow();
const remoteServer = await startRemoteServer(kind);

6
tests/library/browsertype-launch.spec.ts
View file

@ -60,7 +60,7 @@ it('should throw if page argument is passed', async ({ browserType, browserName
});
it('should reject if launched browser fails immediately', async ({ mode, browserType, asset }) => {
it.skip(mode === 'service');
it.skip(mode.startsWith('service'));
let waitError = null;
await browserType.launch({ executablePath: asset('dummy_bad_browser_executable.js') }).catch(e => waitError = e);
@ -68,7 +68,7 @@ it('should reject if launched browser fails immediately', async ({ mode, browser
});
it('should reject if executable path is invalid', async ({ browserType, mode }) => {
it.skip(mode === 'service', 'on service mode we dont allow passing custom executable path');
it.skip(mode.startsWith('service'), 'on service mode we dont allow passing custom executable path');
let waitError = null;
await browserType.launch({ executablePath: 'random-invalid-path' }).catch(e => waitError = e);
expect(waitError.message).toContain('Failed to launch');
@ -103,7 +103,7 @@ it('should report launch log', async ({ browserType, mode }) => {
});
it('should accept objects as options', async ({ mode, browserType }) => {
it.skip(mode === 'service');
it.skip(mode.startsWith('service'));
// @ts-expect-error process is not a real option.
const browser = await browserType.launch({ process });

2
tests/library/capabilities.spec.ts
View file

@ -72,6 +72,7 @@ it('should play video @smoke', async ({ page, asset, browserName, platform, mode
it.fixme(browserName === 'webkit' && platform !== 'darwin');
it.fixme(browserName === 'firefox', 'https://github.com/microsoft/playwright/issues/5721');
it.fixme(browserName === 'webkit' && platform === 'darwin' && parseInt(os.release(), 10) === 20, 'Does not work on BigSur');
it.skip(mode.startsWith('service'));
// Safari only plays mp4 so we test WebKit with an .mp4 clip.
const fileName = browserName === 'webkit' ? 'video_mp4.html' : 'video.html';
@ -86,6 +87,7 @@ it('should play video @smoke', async ({ page, asset, browserName, platform, mode
it('should play webm video @smoke', async ({ page, asset, browserName, platform, mode }) => {
it.fixme(browserName === 'webkit' && platform === 'darwin' && parseInt(os.release(), 10) === 20, 'Does not work on BigSur');
it.fixme(browserName === 'webkit' && platform === 'win32');
it.skip(mode.startsWith('service'));
const absolutePath = asset('video_webm.html');
// Our test server doesn't support range requests required to play on Mac,

2
tests/library/chromium/connect-over-cdp.spec.ts
View file

@ -21,7 +21,7 @@ import fs from 'fs';
import { getUserAgent } from '../../../packages/playwright-core/lib/utils/userAgent';
import { suppressCertificateWarning } from '../../config/utils';
test.skip(({ mode }) => mode === 'service');
test.skip(({ mode }) => mode.startsWith('service'));
test('should connect to an existing cdp session', async ({ browserType, mode }, testInfo) => {
const port = 9339 + testInfo.workerIndex;

2
tests/library/debug-controller.spec.ts
View file

@ -65,7 +65,7 @@ const test = baseTest.extend<Fixtures>({
});
test.slow(true, 'All controller tests are slow');
test.skip(({ mode }) => mode === 'service');
test.skip(({ mode }) => mode.startsWith('service'));
test('should pick element', async ({ backend, connectedBrowser }) => {
const events = [];

14
tests/library/har.spec.ts
View file

@ -53,7 +53,7 @@ it('should have version and creator', async ({ contextFactory, server }, testInf
const log = await getLog();
expect(log.version).toBe('1.2');
expect(log.creator.name).toBe('Playwright');
expect(log.creator.version).toBe(require('../../package.json')['version']);
expect(log.creator.version).toBe(process.env.PW_VERSION_OVERRIDE || require('../../package.json')['version']);
});
it('should have browser', async ({ browserName, browser, contextFactory, server }, testInfo) => {
@ -584,7 +584,7 @@ it('should have connection details', async ({ contextFactory, server, browserNam
const log = await getLog();
const { serverIPAddress, _serverPort: port, _securityDetails: securityDetails } = log.entries[0];
expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
if (mode !== 'service')
if (!mode.startsWith('service'))
expect(port).toBe(server.PORT);
expect(securityDetails).toEqual({});
});
@ -598,7 +598,7 @@ it('should have security details', async ({ contextFactory, httpsServer, browser
const log = await getLog();
const { serverIPAddress, _serverPort: port, _securityDetails: securityDetails } = log.entries[0];
expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
if (mode !== 'service')
if (!mode.startsWith('service'))
expect(port).toBe(httpsServer.PORT);
if (browserName === 'webkit' && platform === 'darwin')
expect(securityDetails).toEqual({ protocol: 'TLS 1.3', subjectName: 'puppeteer-tests', validFrom: 1550084863, validTo: 33086084863 });
@ -620,13 +620,13 @@ it('should have connection details for redirects', async ({ contextFactory, serv
expect(detailsFoo._serverPort).toBeUndefined();
} else {
expect(detailsFoo.serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
if (mode !== 'service')
if (!mode.startsWith('service'))
expect(detailsFoo._serverPort).toBe(server.PORT);
}
const detailsEmpty = log.entries[1];
expect(detailsEmpty.serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
if (mode !== 'service')
if (!mode.startsWith('service'))
expect(detailsEmpty._serverPort).toBe(server.PORT);
});
@ -640,7 +640,7 @@ it('should have connection details for failed requests', async ({ contextFactory
const log = await getLog();
const { serverIPAddress, _serverPort: port } = log.entries[0];
expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
if (mode !== 'service')
if (!mode.startsWith('service'))
expect(port).toBe(server.PORT);
});
@ -648,7 +648,7 @@ it('should return server address directly from response', async ({ page, server,
const response = await page.goto(server.EMPTY_PAGE);
const { ipAddress, port } = await response.serverAddr();
expect(ipAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
if (mode !== 'service')
if (!mode.startsWith('service'))
expect(port).toBe(server.PORT);
});

4
tests/library/inspector/inspectorTest.ts
View file

@ -50,7 +50,7 @@ const playwrightToAutomateInspector = require('../../../packages/playwright-core
export const test = contextTest.extend<CLITestArgs>({
recorderPageGetter: async ({ context, toImpl, mode }, run, testInfo) => {
process.env.PWTEST_RECORDER_PORT = String(10907 + testInfo.workerIndex);
testInfo.skip(mode === 'service');
testInfo.skip(mode.startsWith('service'));
await run(async () => {
while (!toImpl(context).recorderAppForTest)
await new Promise(f => setTimeout(f, 100));
@ -69,7 +69,7 @@ export const test = contextTest.extend<CLITestArgs>({
runCLI: async ({ childProcess, browserName, channel, headless, mode, launchOptions }, run, testInfo) => {
process.env.PWTEST_RECORDER_PORT = String(10907 + testInfo.workerIndex);
testInfo.skip(mode === 'service');
testInfo.skip(mode.startsWith('service'));
await run((cliArgs, { autoExitWhen } = {}) => {
return new CLIMock(childProcess, browserName, channel, headless, cliArgs, launchOptions.executablePath, autoExitWhen);

2
tests/library/launcher.spec.ts
View file

@ -43,7 +43,7 @@ it('should kill browser process on timeout after close', async ({ browserType, m
it('should throw a friendly error if its headed and there is no xserver on linux running', async ({ mode, browserType, platform }) => {
it.skip(platform !== 'linux');
it.skip(mode === 'service');
it.skip(mode.startsWith('service'));
const error: Error = await browserType.launch({
headless: false,

24
tests/library/playwright.config.ts
View file

@ -34,7 +34,7 @@ const getExecutablePath = (browserName: BrowserName) => {
return process.env.WKPATH;
};
const mode: TestModeName = (process.env.PWTEST_MODE ?? 'default') as ('default' | 'driver' | 'service');
const mode = (process.env.PWTEST_MODE ?? 'default') as TestModeName;
const headed = process.argv.includes('--headed');
const channel = process.env.PWTEST_CHANNEL as any;
const video = !!process.env.PWTEST_VIDEO;
@ -52,6 +52,22 @@ const reporters = () => {
];
return result;
};
const os: 'linux' | 'windows' = 'linux'; // use linux
const runId = new Date().toISOString(); // name the test run
let connectOptions: any;
if (mode === 'service')
connectOptions = { wsEndpoint: 'ws://localhost:3333/' };
if (mode === 'service2') {
process.env.PW_VERSION_OVERRIDE = '1.36.0';
connectOptions = {
wsEndpoint: `${process.env.PLAYWRIGHT_SERVICE_URL}?accessKey=${process.env.PLAYWRIGHT_SERVICE_ACCESS_KEY}&cap=${JSON.stringify({ os, runId })}`,
timeout: 3 * 60 * 1000,
_exposeNetwork: '<loopback>',
};
}
const config: Config<CoverageWorkerOptions & PlaywrightWorkerOptions & PlaywrightTestOptions & TestModeWorkerOptions> = {
testDir,
outputDir,
@ -60,7 +76,7 @@ const config: Config<CoverageWorkerOptions & PlaywrightWorkerOptions & Playwrigh
toHaveScreenshot: { _comparator: 'ssim-cie94' } as any,
toMatchSnapshot: { _comparator: 'ssim-cie94' } as any,
},
maxFailures: 100,
maxFailures: 200,
timeout: video ? 60000 : 30000,
globalTimeout: 5400000,
workers: process.env.CI ? 2 : undefined,
@ -70,9 +86,7 @@ const config: Config<CoverageWorkerOptions & PlaywrightWorkerOptions & Playwrigh
reporter: reporters(),
projects: [],
use: {
connectOptions: mode === 'service' ? {
wsEndpoint: 'ws://localhost:3333/',
} : undefined,
connectOptions,
},
webServer: mode === 'service' ? {
command: 'npx playwright run-server --port=3333',

2
tests/library/proxy.spec.ts
View file

@ -18,7 +18,7 @@ import { playwrightTest as it, expect } from '../config/browserTest';
import socks from 'socksv5';
import net from 'net';
it.skip(({ mode }) => mode === 'service');
it.skip(({ mode }) => mode.startsWith('service'));
it('should throw for bad server value', async ({ browserType }) => {
const error = await browserType.launch({

8
tests/library/tracing.spec.ts
View file

@ -183,7 +183,7 @@ test('should collect two traces', async ({ context, page, server }, testInfo) =>
});
test('should respect tracesDir and name', async ({ browserType, server, mode }, testInfo) => {
test.skip(mode === 'service', 'Service ignores tracesDir');
test.skip(mode.startsWith('service'), 'Service ignores tracesDir');
const tracesDir = testInfo.outputPath('traces');
const browser = await browserType.launch({ tracesDir });
@ -236,7 +236,7 @@ test('should respect tracesDir and name', async ({ browserType, server, mode },
}
});
test('should not include trace resources from the provious chunks', async ({ context, page, server, browserName }, testInfo) => {
test('should not include trace resources from the previous chunks', async ({ context, page, server, browserName }, testInfo) => {
test.skip(browserName !== 'chromium', 'The number of screenshots is flaky in non-Chromium');
await context.tracing.start({ screenshots: true, snapshots: true, sources: true });
@ -638,7 +638,7 @@ test('should store postData for global request', async ({ request, server }, tes
});
test('should not flush console events', async ({ context, page, mode }, testInfo) => {
test.skip(mode === 'service', 'Uses artifactsFolderName');
test.skip(mode.startsWith('service'), 'Uses artifactsFolderName');
const testId = test.info().testId;
await context.tracing.start({ name: testId });
const promise = new Promise<void>(f => {
@ -704,7 +704,7 @@ test('should flush console events on tracing stop', async ({ context, page }, te
});
test('should not emit after w/o before', async ({ browserType, mode }, testInfo) => {
test.skip(mode === 'service', 'Service ignores tracesDir');
test.skip(mode.startsWith('service'), 'Service ignores tracesDir');
const tracesDir = testInfo.outputPath('traces');
const browser = await browserType.launch({ tracesDir });

4
tests/page/frame-goto.spec.ts
View file

@ -38,9 +38,9 @@ it('should reject when frame detaches', async ({ page, server, browserName }) =>
await page.$eval('iframe', frame => frame.remove());
const error = await navigationPromise;
if (browserName === 'chromium')
expect(error.message.includes('net::ERR_ABORTED') || error.message.includes('Frame was detached')).toBe(true);
expect(error.message.includes('net::ERR_ABORTED') || error.message.toLowerCase().includes('frame was detached')).toBe(true);
else
expect(error.message).toContain('Frame was detached');
expect(error.message.toLowerCase()).toContain('frame was detached');
});
it('should continue after client redirect', async ({ page, server, isAndroid }) => {

25
tests/page/page-goto.spec.ts
View file

@ -24,8 +24,9 @@ it('should work @smoke', async ({ page, server }) => {
expect(page.url()).toBe(server.EMPTY_PAGE);
});
it('should work with file URL', async ({ page, asset, isAndroid }) => {
it('should work with file URL', async ({ page, asset, isAndroid, mode }) => {
it.skip(isAndroid, 'No files on Android');
it.skip(mode.startsWith('service'));
const fileurl = url.pathToFileURL(asset('empty.html')).href;
await page.goto(fileurl);
@ -33,8 +34,9 @@ it('should work with file URL', async ({ page, asset, isAndroid }) => {
expect(page.frames().length).toBe(1);
});
it('should work with file URL with subframes', async ({ page, asset, isAndroid }) => {
it('should work with file URL with subframes', async ({ page, asset, isAndroid, mode }) => {
it.skip(isAndroid, 'No files on Android');
it.skip(mode.startsWith('service'));
const fileurl = url.pathToFileURL(asset('frames/two-frames.html')).href;
await page.goto(fileurl);
@ -300,14 +302,21 @@ it('should throw if networkidle2 is passed as an option', async ({ page, server
it('should fail when main resources failed to load', async ({ page, browserName, isWindows, mode }) => {
let error = null;
await page.goto('http://localhost:44123/non-existing-url').catch(e => error = e);
if (browserName === 'chromium')
expect(error.message).toContain('net::ERR_CONNECTION_REFUSED');
else if (browserName === 'webkit' && isWindows)
if (browserName === 'chromium') {
if (mode === 'service2')
expect(error.message).toContain('net::ERR_SOCKS_CONNECTION_FAILED');
else
expect(error.message).toContain('net::ERR_CONNECTION_REFUSED');
} else if (browserName === 'webkit' && isWindows) {
expect(error.message).toContain(`Couldn\'t connect to server`);
else if (browserName === 'webkit')
expect(error.message).toContain('Could not connect');
else
} else if (browserName === 'webkit') {
if (mode === 'service2')
expect(error.message).toContain('Connection refused');
else
expect(error.message).toContain('Could not connect');
} else {
expect(error.message).toContain('NS_ERROR_CONNECTION_REFUSED');
}
});
it('should fail when exceeding maximum navigation timeout', async ({ page, server, playwright }) => {

3
tests/page/page-history.spec.ts
View file

@ -52,9 +52,10 @@ it('page.goBack should work with HistoryAPI', async ({ page, server }) => {
expect(page.url()).toBe(server.PREFIX + '/first.html');
});
it('page.goBack should work for file urls', async ({ page, server, asset, browserName, platform, isAndroid }) => {
it('page.goBack should work for file urls', async ({ page, server, asset, browserName, platform, isAndroid, mode }) => {
it.fail(browserName === 'webkit' && platform === 'darwin', 'WebKit embedder fails to go back/forward to the file url.');
it.skip(isAndroid, 'No files on Android');
it.skip(mode.startsWith('service'));
const url1 = url.pathToFileURL(asset('consolelog.html')).href;
const url2 = server.PREFIX + '/consolelog.html';

3
tests/page/page-network-response.spec.ts
View file

@ -270,9 +270,10 @@ it('should behave the same way for headers and allHeaders', async ({ page, serve
expect(allHeaders['name-b']).toEqual('v4');
});
it('should provide a Response with a file URL', async ({ page, asset, isAndroid, isElectron, isWindows, browserName, browserMajorVersion }) => {
it('should provide a Response with a file URL', async ({ page, asset, isAndroid, isElectron, isWindows, browserName, browserMajorVersion, mode }) => {
it.skip(isAndroid, 'No files on Android');
it.skip(browserName === 'firefox', 'Firefox does return null for file:// URLs');
it.skip(mode.startsWith('service'));
const fileurl = url.pathToFileURL(asset('frames/two-frames.html')).href;
const response = await page.goto(fileurl);

12
tests/page/page-set-input-files.spec.ts
View file

@ -37,10 +37,12 @@ it('should upload the file', async ({ page, server, asset }) => {
}, input)).toBe('contents of the file');
});
it('should upload large file', async ({ page, server, browserName, isMac, isAndroid }, testInfo) => {
it('should upload large file', async ({ page, server, browserName, isMac, isAndroid, mode }, testInfo) => {
it.skip(browserName === 'webkit' && isMac && parseInt(os.release(), 10) < 20, 'WebKit for macOS 10.15 is frozen and does not have corresponding protocol features.');
it.skip(isAndroid);
it.skip(mode.startsWith('service'));
it.slow();
await page.goto(server.PREFIX + '/input/fileupload.html');
const uploadFile = testInfo.outputPath('200MB.zip');
const str = 'A'.repeat(4 * 1024);
@ -85,10 +87,12 @@ it('should upload large file', async ({ page, server, browserName, isMac, isAndr
await Promise.all([uploadFile, file1.filepath].map(fs.promises.unlink));
});
it('should upload multiple large files', async ({ page, server, browserName, isMac, isAndroid }, testInfo) => {
it('should upload multiple large files', async ({ page, server, browserName, isMac, isAndroid, mode }, testInfo) => {
it.skip(browserName === 'webkit' && isMac && parseInt(os.release(), 10) < 20, 'WebKit for macOS 10.15 is frozen and does not have corresponding protocol features.');
it.skip(isAndroid);
it.skip(mode.startsWith('service'));
it.slow();
const filesCount = 10;
await page.goto(server.PREFIX + '/input/fileupload-multi.html');
const uploadFile = testInfo.outputPath('50MB_1.zip');
@ -123,10 +127,12 @@ it('should upload multiple large files', async ({ page, server, browserName, isM
await Promise.all(uploadFiles.map(path => fs.promises.unlink(path)));
});
it('should upload large file with relative path', async ({ page, server, browserName, isMac, isAndroid }, testInfo) => {
it('should upload large file with relative path', async ({ page, server, browserName, isMac, isAndroid, mode }, testInfo) => {
it.skip(browserName === 'webkit' && isMac && parseInt(os.release(), 10) < 20, 'WebKit for macOS 10.15 is frozen and does not have corresponding protocol features.');
it.skip(isAndroid);
it.skip(mode.startsWith('service'));
it.slow();
await page.goto(server.PREFIX + '/input/fileupload.html');
const uploadFile = testInfo.outputPath('200MB.zip');
const str = 'A'.repeat(4 * 1024);