diff --git a/package-lock.json b/package-lock.json index 744f2ba9d4..1f55c38b67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,7 +36,6 @@ "mime": "^2.4.6", "minimatch": "^3.0.3", "ms": "^2.1.2", - "node-fetch": "^2.6.1", "pirates": "^4.0.1", "pixelmatch": "^5.2.1", "pngjs": "^5.0.0", @@ -6967,14 +6966,6 @@ "integrity": "sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==", "dev": true }, - "node_modules/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", - "engines": { - "node": "4.x || >=6.0.0" - } - }, "node_modules/node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", @@ -8596,7 +8587,6 @@ }, "node_modules/socksv5/node_modules/ipv6": { "version": "3.1.1", - "resolved": "https://registry.npmjs.org/ipv6/-/ipv6-3.1.1.tgz", "dev": true, "inBundle": true, "license": "MIT", @@ -8615,7 +8605,6 @@ }, "node_modules/socksv5/node_modules/ipv6/node_modules/sprintf": { "version": "0.1.3", - "resolved": "https://registry.npmjs.org/sprintf/-/sprintf-0.1.3.tgz", "dev": true, "inBundle": true, "engines": { @@ -15961,11 +15950,6 @@ } } }, - "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" - }, "node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", @@ -17275,7 +17259,6 @@ "dependencies": { "ipv6": { "version": "3.1.1", - "resolved": "https://registry.npmjs.org/ipv6/-/ipv6-3.1.1.tgz", "bundled": true, "dev": true, "requires": { @@ -17286,7 +17269,6 @@ "dependencies": { "sprintf": { "version": "0.1.3", - "resolved": "https://registry.npmjs.org/sprintf/-/sprintf-0.1.3.tgz", "bundled": true, "dev": true } diff --git a/package.json b/package.json index c8cd015bb8..0db099bb72 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,6 @@ "mime": "^2.4.6", "minimatch": "^3.0.3", "ms": "^2.1.2", - "node-fetch": "^2.6.1", "pirates": "^4.0.1", "pixelmatch": "^5.2.1", "pngjs": "^5.0.0", @@ -91,7 +90,6 @@ "@types/mime": "^2.0.3", "@types/minimatch": "^3.0.3", "@types/node": "^10.17.28", - "@types/node-fetch": "^2.5.12", "@types/pixelmatch": "^5.2.1", "@types/pngjs": "^3.4.2", "@types/progress": "^2.0.3", diff --git a/packages/build_package.js b/packages/build_package.js index e4fe513f7b..c2d0447215 100755 --- a/packages/build_package.js +++ b/packages/build_package.js @@ -72,7 +72,6 @@ const DEPENDENCIES = [ 'https-proxy-agent', 'jpeg-js', 'mime', - 'node-fetch', 'pngjs', 'progress', 'proper-lockfile', diff --git a/src/server/fetch.ts b/src/server/fetch.ts index 23ada7257c..631777348e 100644 --- a/src/server/fetch.ts +++ b/src/server/fetch.ts @@ -15,8 +15,9 @@ */ import { HttpsProxyAgent } from 'https-proxy-agent'; -import nodeFetch from 'node-fetch'; -import * as url from 'url'; +import url from 'url'; +import * as http from 'http'; +import * as https from 'https'; import { BrowserContext } from './browserContext'; import * as types from './types'; @@ -45,54 +46,130 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet } // TODO(https://github.com/microsoft/playwright/issues/8381): set user agent - const response = await nodeFetch(params.url, { + const {fetchResponse, setCookie} = await sendRequest(new URL(params.url), { method: params.method, headers: params.headers, - body: params.postData, - agent - }); - const body = await response.buffer(); - const setCookies = response.headers.raw()['set-cookie']; - if (setCookies) { - const url = new URL(response.url); - // https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4 - const defaultPath = '/' + url.pathname.split('/').slice(0, -1).join('/'); - const cookies: types.SetNetworkCookieParam[] = []; - for (const header of setCookies) { - // Decode cookie value? - const cookie: types.SetNetworkCookieParam | null = parseCookie(header); - if (!cookie) - continue; - if (!cookie.domain) - cookie.domain = url.hostname; - if (!canSetCookie(cookie.domain!, url.hostname)) - continue; - // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4 - if (!cookie.path || !cookie.path.startsWith('/')) - cookie.path = defaultPath; - cookies.push(cookie); - } - if (cookies.length) - await context.addCookies(cookies); - } - - const headers: types.HeadersArray = []; - for (const [name, value] of response.headers.entries()) - headers.push({ name, value }); - return { - fetchResponse: { - url: response.url, - status: response.status, - statusText: response.statusText, - headers, - body - } - }; + agent, + maxRedirects: 20 + }, params.postData); + if (setCookie) + await updateCookiesFromHeader(context, fetchResponse.url, setCookie); + return { fetchResponse }; } catch (e) { return { error: String(e) }; } } +async function updateCookiesFromHeader(context: BrowserContext, responseUrl: string, setCookie: string[]) { + const url = new URL(responseUrl); + // https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4 + const defaultPath = '/' + url.pathname.split('/').slice(0, -1).join('/'); + const cookies: types.SetNetworkCookieParam[] = []; + for (const header of setCookie) { + // Decode cookie value? + const cookie: types.SetNetworkCookieParam | null = parseCookie(header); + if (!cookie) + continue; + if (!cookie.domain) + cookie.domain = url.hostname; + if (!canSetCookie(cookie.domain!, url.hostname)) + continue; + // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4 + if (!cookie.path || !cookie.path.startsWith('/')) + cookie.path = defaultPath; + cookies.push(cookie); + } + if (cookies.length) + await context.addCookies(cookies); +} + +type Response = { + fetchResponse: types.FetchResponse, + setCookie?: string[] +}; + +async function sendRequest(url: URL, options: http.RequestOptions & { maxRedirects: number }, postData?: Buffer): Promise{ + return new Promise((fulfill, reject) => { + const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest) + = (url.protocol === 'https:' ? https : http).request; + const request = requestConstructor(url, options, response => { + if (redirectStatus.includes(response.statusCode!)) { + if (!options.maxRedirects) { + reject(new Error('Max redirect count exceeded')); + request.abort(); + return; + } + const redirectOptions: http.RequestOptions & { maxRedirects: number } = { + method: options.method, + headers: { ...options.headers }, + agent: options.agent, + maxRedirects: options.maxRedirects - 1, + }; + + // HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch) + const status = response.statusCode!; + const method = redirectOptions.method!; + if ((status === 301 || status === 302) && method === 'POST' || + status === 303 && !['GET', 'HEAD'].includes(method)) { + redirectOptions.method = 'GET'; + postData = undefined; + delete redirectOptions.headers?.[`content-encoding`]; + delete redirectOptions.headers?.[`content-language`]; + delete redirectOptions.headers?.[`content-location`]; + delete redirectOptions.headers?.[`content-type`]; + } + + // TODO: set-cookie from response, add cookie from the context. + + // HTTP-redirect fetch step 4: If locationURL is null, then return response. + if (response.headers.location) { + const locationURL = new URL(response.headers.location, url); + fulfill(sendRequest(locationURL, redirectOptions, postData)); + request.abort(); + return; + } + } + const chunks: Buffer[] = []; + response.on('data', chunk => chunks.push(chunk)); + response.on('end', () => { + const body = Buffer.concat(chunks); + fulfill({ + fetchResponse: { + url: response.url || url.toString(), + status: response.statusCode || 0, + statusText: response.statusMessage || '', + headers: flattenHeaders(response.headers), + body + }, + setCookie: response.headers['set-cookie'] + }); + }); + response.on('error',reject); + }); + request.on('error', reject); + if (postData) + request.write(postData); + request.end(); + }); +} + +function flattenHeaders(headers: http.IncomingHttpHeaders): types.HeadersArray { + const result: types.HeadersArray = []; + for (const [name, values] of Object.entries(headers)) { + if (values === undefined) + continue; + if (typeof values === 'string') { + result.push({name, value: values as string}); + } else { + for (const value of values) + result.push({name, value}); + } + } + return result; +} + +const redirectStatus = [301, 302, 303, 307, 308]; + function canSetCookie(cookieDomain: string, hostname: string) { // TODO: check public suffix list? hostname = '.' + hostname; @@ -101,7 +178,6 @@ function canSetCookie(cookieDomain: string, hostname: string) { return hostname.endsWith(cookieDomain); } - function parseCookie(header: string) { const pairs = header.split(';').filter(s => s.trim().length > 0).map(p => p.split('=').map(s => s.trim())); if (!pairs.length) diff --git a/src/server/types.ts b/src/server/types.ts index 5a3f557d5d..985ca12806 100644 --- a/src/server/types.ts +++ b/src/server/types.ts @@ -379,9 +379,6 @@ export type FetchResponse = { url: string, status: number, statusText: string, - headers: { - name: string, - value: string, - }[], + headers: HeadersArray, body: Buffer, };