diff --git a/src/server/fetch.ts b/src/server/fetch.ts index 1a02fffb41..340fbb0872 100644 --- a/src/server/fetch.ts +++ b/src/server/fetch.ts @@ -16,10 +16,12 @@ import { HttpsProxyAgent } from 'https-proxy-agent'; import url from 'url'; +import zlib from 'zlib'; import * as http from 'http'; import * as https from 'https'; import { BrowserContext } from './browserContext'; import * as types from './types'; +import { pipeline, Readable, Transform } from 'stream'; export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: types.FetchResponse, error?: string}> { try { @@ -30,7 +32,7 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet } headers['user-agent'] ??= context._options.userAgent || context._browser.userAgent(); headers['accept'] ??= '*/*'; - headers['accept-encoding'] ??= 'gzip,deflate'; + headers['accept-encoding'] ??= 'gzip,deflate,br'; if (context._options.extraHTTPHeaders) { for (const {name, value} of context._options.extraHTTPHeaders) @@ -149,9 +151,31 @@ async function sendRequest(context: BrowserContext, url: URL, options: http.Requ return; } } + response.on('aborted', () => reject(new Error('aborted'))); + + let body: Readable = response; + let transform: Transform | undefined; + const encoding = response.headers['content-encoding']; + if (encoding === 'gzip' || encoding === 'x-gzip') { + transform = zlib.createGunzip({ + flush: zlib.constants.Z_SYNC_FLUSH, + finishFlush: zlib.constants.Z_SYNC_FLUSH + }); + } else if (encoding === 'br') { + transform = zlib.createBrotliDecompress(); + } else if (encoding === 'deflate') { + transform = zlib.createInflate(); + } + if (transform) { + body = pipeline(response, transform, e => { + if (e) + reject(new Error(`failed to decompress '${encoding}' encoding: ${e}`)); + }); + } + const chunks: Buffer[] = []; - response.on('data', chunk => chunks.push(chunk)); - response.on('end', () => { + body.on('data', chunk => chunks.push(chunk)); + body.on('end', () => { const body = Buffer.concat(chunks); fulfill({ url: response.url || url.toString(), @@ -161,8 +185,7 @@ async function sendRequest(context: BrowserContext, url: URL, options: http.Requ body }); }); - response.on('aborted', () => reject(new Error('aborted'))); - response.on('error',reject); + body.on('error',reject); }); request.on('error', reject); if (postData) diff --git a/tests/browsercontext-fetch.spec.ts b/tests/browsercontext-fetch.spec.ts index cad4c8dc55..9e68c2c166 100644 --- a/tests/browsercontext-fetch.spec.ts +++ b/tests/browsercontext-fetch.spec.ts @@ -15,6 +15,8 @@ */ import http from 'http'; +import zlib from 'zlib'; +import { pipeline } from 'stream'; import { contextTest as it, expect } from './config/browserTest'; it.skip(({ mode }) => mode !== 'default'); @@ -339,7 +341,7 @@ it('should add default headers', async ({context, server, page}) => { expect(request.headers['accept']).toBe('*/*'); const userAgent = await page.evaluate(() => navigator.userAgent); expect(request.headers['user-agent']).toBe(userAgent); - expect(request.headers['accept-encoding']).toBe('gzip,deflate'); + expect(request.headers['accept-encoding']).toBe('gzip,deflate,br'); }); it('should add default headers to redirects', async ({context, server, page}) => { @@ -352,7 +354,7 @@ it('should add default headers to redirects', async ({context, server, page}) => expect(request.headers['accept']).toBe('*/*'); const userAgent = await page.evaluate(() => navigator.userAgent); expect(request.headers['user-agent']).toBe(userAgent); - expect(request.headers['accept-encoding']).toBe('gzip,deflate'); + expect(request.headers['accept-encoding']).toBe('gzip,deflate,br'); }); it('should allow to override default headers', async ({context, server, page}) => { @@ -444,3 +446,112 @@ it('should resolve url relative to baseURL', async function({browser, server, co const response = await context._fetch('/empty.html'); expect(response.url()).toBe(server.EMPTY_PAGE); }); + +it('should support gzip compression', async function({context, server}) { + server.setRoute('/compressed', (req, res) => { + res.writeHead(200, { + 'Content-Encoding': 'gzip', + 'Content-Type': 'text/plain', + }); + + const gzip = zlib.createGzip(); + pipeline(gzip, res, err => { + if (err) + console.log(`Server error: ${err}`); + }); + gzip.write('Hello, world!'); + gzip.end(); + }); + + // @ts-expect-error + const response = await context._fetch(server.PREFIX + '/compressed'); + expect(await response.text()).toBe('Hello, world!'); +}); + +it('should throw informatibe error on corrupted gzip body', async function({context, server}) { + server.setRoute('/corrupted', (req, res) => { + res.writeHead(200, { + 'Content-Encoding': 'gzip', + 'Content-Type': 'text/plain', + }); + res.write('Hello, world!'); + res.end(); + }); + + // @ts-expect-error + const error = await context._fetch(server.PREFIX + '/corrupted').catch(e => e); + expect(error.message).toContain(`failed to decompress 'gzip' encoding`); +}); + +it('should support brotli compression', async function({context, server}) { + server.setRoute('/compressed', (req, res) => { + res.writeHead(200, { + 'Content-Encoding': 'br', + 'Content-Type': 'text/plain', + }); + + const brotli = zlib.createBrotliCompress(); + pipeline(brotli, res, err => { + if (err) + console.log(`Server error: ${err}`); + }); + brotli.write('Hello, world!'); + brotli.end(); + }); + + // @ts-expect-error + const response = await context._fetch(server.PREFIX + '/compressed'); + expect(await response.text()).toBe('Hello, world!'); +}); + +it('should throw informatibe error on corrupted brotli body', async function({context, server}) { + server.setRoute('/corrupted', (req, res) => { + res.writeHead(200, { + 'Content-Encoding': 'br', + 'Content-Type': 'text/plain', + }); + res.write('Hello, world!'); + res.end(); + }); + + // @ts-expect-error + const error = await context._fetch(server.PREFIX + '/corrupted').catch(e => e); + expect(error.message).toContain(`failed to decompress 'br' encoding`); +}); + +it('should support deflate compression', async function({context, server}) { + server.setRoute('/compressed', (req, res) => { + res.writeHead(200, { + 'Content-Encoding': 'deflate', + 'Content-Type': 'text/plain', + }); + + const deflate = zlib.createDeflate(); + pipeline(deflate, res, err => { + if (err) + console.log(`Server error: ${err}`); + }); + deflate.write('Hello, world!'); + deflate.end(); + }); + + // @ts-expect-error + const response = await context._fetch(server.PREFIX + '/compressed'); + expect(await response.text()).toBe('Hello, world!'); +}); + +it('should throw informatibe error on corrupted deflate body', async function({context, server}) { + server.setRoute('/corrupted', (req, res) => { + res.writeHead(200, { + 'Content-Encoding': 'deflate', + 'Content-Type': 'text/plain', + }); + res.write('Hello, world!'); + res.end(); + }); + + // @ts-expect-error + const error = await context._fetch(server.PREFIX + '/corrupted').catch(e => e); + expect(error.message).toContain(`failed to decompress 'deflate' encoding`); +}); +