From 2d428c8a4e468629cf146388da393919e14ce7cf Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 29 Sep 2021 17:15:32 -0700 Subject: [PATCH] feat(fetch): store cookies between requests (#9221) --- src/server/cookieStore.ts | 133 ++++++++++++++++++++++++ src/server/fetch.ts | 46 ++++----- tests/global-fetch-cookie.spec.ts | 164 ++++++++++++++++++++++++++++++ 3 files changed, 320 insertions(+), 23 deletions(-) create mode 100644 src/server/cookieStore.ts create mode 100644 tests/global-fetch-cookie.spec.ts diff --git a/src/server/cookieStore.ts b/src/server/cookieStore.ts new file mode 100644 index 0000000000..5c9518986d --- /dev/null +++ b/src/server/cookieStore.ts @@ -0,0 +1,133 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as types from './types'; + +class Cookie { + private _raw: types.NetworkCookie; + constructor(data: types.NetworkCookie) { + this._raw = data; + } + + name(): string { + return this._raw.name; + } + + // https://datatracker.ietf.org/doc/html/rfc6265#section-5.4 + matches(url: URL): boolean { + if (this._raw.secure && url.protocol !== 'https:') + return false; + if (!domainMatches(url.hostname, this._raw.domain)) + return false; + if (!pathMatches(url.pathname, this._raw.path)) + return false; + return true; + } + + equals(other: Cookie) { + return this._raw.name === other._raw.name && + this._raw.domain === other._raw.domain && + this._raw.path === other._raw.path; + } + + networkCookie(): types.NetworkCookie { + return this._raw; + } + + updateExpiresFrom(other: Cookie) { + this._raw.expires = other._raw.expires; + } + + expired() { + if (this._raw.expires === -1) + return false; + return this._raw.expires * 1000 < Date.now(); + } +} + +export class CookieStore { + private readonly _nameToCookies: Map> = new Map(); + + addCookies(cookies: types.NetworkCookie[]) { + for (const cookie of cookies) + this._addCookie(new Cookie(cookie)); + } + + cookies(url: URL): types.NetworkCookie[] { + const result = []; + for (const cookie of this._allCookies()) { + if (cookie.matches(url)) + result.push(cookie.networkCookie()); + } + return result; + } + + private _addCookie(cookie: Cookie) { + if (cookie.expired()) + return; + let set = this._nameToCookies.get(cookie.name()); + if (!set) { + set = new Set(); + this._nameToCookies.set(cookie.name(), set); + } + CookieStore.pruneExpired(set); + // https://datatracker.ietf.org/doc/html/rfc6265#section-5.3 + for (const other of set) { + if (other.equals(cookie)) { + cookie.updateExpiresFrom(other); + set.delete(other); + } + } + set.add(cookie); + } + + private *_allCookies(): IterableIterator { + for (const [name, cookies] of this._nameToCookies) { + CookieStore.pruneExpired(cookies); + for (const cookie of cookies) + yield cookie; + if (cookies.size === 0) + this._nameToCookies.delete(name); + } + } + + private static pruneExpired(cookies: Set) { + for (const cookie of cookies) { + if (cookie.expired()) + cookies.delete(cookie); + } + } +} + +export function domainMatches(value: string, domain: string): boolean { + if (value === domain) + return true; + // Only strict match is allowed if domain doesn't start with '.' (host-only-flag is true in the spec) + if (!domain.startsWith('.')) + return false; + value = '.' + value; + return value.endsWith(domain); +} + +function pathMatches(value: string, path: string): boolean { + if (value === path) + return true; + if (!value.endsWith('/')) + value = value + '/'; + if (!path.endsWith('/')) + path = path + '/'; + return value.startsWith(path); +} diff --git a/src/server/fetch.ts b/src/server/fetch.ts index f7afbafbde..1c02174d2a 100644 --- a/src/server/fetch.ts +++ b/src/server/fetch.ts @@ -23,8 +23,9 @@ import zlib from 'zlib'; import { HTTPCredentials } from '../../types/types'; import { NameValue, NewRequestOptions } from '../common/types'; import { TimeoutSettings } from '../utils/timeoutSettings'; -import { createGuid, getPlaywrightVersion, isFilePayload, monotonicTime } from '../utils/utils'; +import { assert, createGuid, getPlaywrightVersion, isFilePayload, monotonicTime } from '../utils/utils'; import { BrowserContext } from './browserContext'; +import { CookieStore, domainMatches } from './cookieStore'; import { MultipartFormData } from './formData'; import { SdkObject } from './instrumentation'; import { Playwright } from './playwright'; @@ -73,8 +74,8 @@ export abstract class FetchRequest extends SdkObject { abstract dispose(): void; abstract _defaultOptions(): FetchRequestOptions; - abstract _addCookies(cookies: types.SetNetworkCookieParam[]): Promise; - abstract _cookies(url: string): Promise; + abstract _addCookies(cookies: types.NetworkCookie[]): Promise; + abstract _cookies(url: URL): Promise; private _storeResponseBody(body: Buffer): string { const uid = createGuid(); @@ -155,15 +156,18 @@ export abstract class FetchRequest extends SdkObject { const url = new URL(responseUrl); // https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4 const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/'); - const cookies: types.SetNetworkCookieParam[] = []; + const cookies: types.NetworkCookie[] = []; for (const header of setCookie) { // Decode cookie value? - const cookie: types.SetNetworkCookieParam | null = parseCookie(header); + const cookie: types.NetworkCookie | null = parseCookie(header); if (!cookie) continue; + // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.3 if (!cookie.domain) cookie.domain = url.hostname; - if (!canSetCookie(cookie.domain!, url.hostname)) + else + assert(cookie.domain.startsWith('.')); + if (!domainMatches(url.hostname, cookie.domain!)) continue; // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4 if (!cookie.path || !cookie.path.startsWith('/')) @@ -177,7 +181,7 @@ export abstract class FetchRequest extends SdkObject { private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) { if (options.headers!['cookie'] !== undefined) return; - const cookies = await this._cookies(url.toString()); + const cookies = await this._cookies(url); if (cookies.length) { const valueArray = cookies.map(c => `${c.name}=${c.value}`); options.headers!['cookie'] = valueArray.join('; '); @@ -326,17 +330,18 @@ export class BrowserContextFetchRequest extends FetchRequest { }; } - async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise { + async _addCookies(cookies: types.NetworkCookie[]): Promise { await this._context.addCookies(cookies); } - async _cookies(url: string): Promise { - return await this._context.cookies(url); + async _cookies(url: URL): Promise { + return await this._context.cookies(url.toString()); } } export class GlobalFetchRequest extends FetchRequest { + private readonly _cookieStore: CookieStore = new CookieStore(); private readonly _options: FetchRequestOptions; constructor(playwright: Playwright, options: Omit & { extraHTTPHeaders?: NameValue[] }) { super(playwright); @@ -370,11 +375,12 @@ export class GlobalFetchRequest extends FetchRequest { return this._options; } - async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise { + async _addCookies(cookies: types.NetworkCookie[]): Promise { + this._cookieStore.addCookies(cookies); } - async _cookies(url: string): Promise { - return []; + async _cookies(url: URL): Promise { + return this._cookieStore.cookies(url); } } @@ -387,15 +393,7 @@ function toHeadersArray(rawHeaders: string[]): types.HeadersArray { const redirectStatus = [301, 302, 303, 307, 308]; -function canSetCookie(cookieDomain: string, hostname: string) { - // TODO: check public suffix list? - hostname = '.' + hostname; - if (!cookieDomain.startsWith('.')) - cookieDomain = '.' + cookieDomain; - return hostname.endsWith(cookieDomain); -} - -function parseCookie(header: string) { +function parseCookie(header: string): types.NetworkCookie | null { const pairs = header.split(';').filter(s => s.trim().length > 0).map(p => p.split('=').map(s => s.trim())); if (!pairs.length) return null; @@ -424,7 +422,9 @@ function parseCookie(header: string) { cookie.expires = Date.now() / 1000 + maxAgeSec; break; case 'domain': - cookie.domain = value || ''; + cookie.domain = value.toLocaleLowerCase() || ''; + if (cookie.domain && !cookie.domain.startsWith('.')) + cookie.domain = '.' + cookie.domain; break; case 'path': cookie.path = value || ''; diff --git a/tests/global-fetch-cookie.spec.ts b/tests/global-fetch-cookie.spec.ts new file mode 100644 index 0000000000..c1bbc847f1 --- /dev/null +++ b/tests/global-fetch-cookie.spec.ts @@ -0,0 +1,164 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import http from 'http'; +import { FetchRequest } from '../index'; +import { expect, playwrightTest } from './config/browserTest'; + +export type GlobalFetchFixtures = { + request: FetchRequest; + }; + +const it = playwrightTest.extend({ + request: async ({ playwright }, use) => { + const request = await playwright._newRequest({ ignoreHTTPSErrors: true }); + await use(request); + await request.dispose(); + }, +}); + +it.skip(({ mode }) => mode !== 'default'); + +let prevAgent: http.Agent; +it.beforeAll(() => { + prevAgent = http.globalAgent; + http.globalAgent = new http.Agent({ + // @ts-expect-error + lookup: (hostname, options, callback) => { + if (hostname === 'localhost' || hostname.endsWith('one.com') || hostname.endsWith('two.com')) + callback(null, '127.0.0.1', 4); + else + throw new Error(`Failed to resolve hostname: ${hostname}`); + } + }); +}); + +it.afterAll(() => { + http.globalAgent = prevAgent; +}); + +it('should store cookie from Set-Cookie header', async ({ request, server }) => { + server.setRoute('/setcookie.html', (req, res) => { + res.setHeader('Set-Cookie', ['a=b', 'c=d; max-age=3600; domain=b.one.com; path=/input', 'e=f; domain=b.one.com; path=/input/subfolder']); + res.end(); + }); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + const [serverRequest] = await Promise.all([ + server.waitForRequest('/input/button.html'), + request.get(`http://b.one.com:${server.PORT}/input/button.html`) + ]); + expect(serverRequest.headers.cookie).toBe('c=d'); +}); + +it('should filter outgoing cookies by path', async ({ request, server }) => { + server.setRoute('/setcookie.html', (req, res) => { + res.setHeader('Set-Cookie', ['a=v; path=/input/subfolder', 'b=v; path=/input', 'c=v;']); + res.end(); + }); + await request.get(`${server.PREFIX}/setcookie.html`); + const [serverRequest] = await Promise.all([ + server.waitForRequest('/input/button.html'), + request.get(`${server.PREFIX}/input/button.html`) + ]); + expect(serverRequest.headers.cookie).toBe('b=v; c=v'); +}); + +it('should filter outgoing cookies by domain', async ({ request, server }) => { + server.setRoute('/setcookie.html', (req, res) => { + res.setHeader('Set-Cookie', ['a=v; domain=one.com', 'b=v; domain=.b.one.com', 'c=v; domain=other.com']); + res.end(); + }); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + const [serverRequest] = await Promise.all([ + server.waitForRequest('/empty.html'), + request.get(`http://www.b.one.com:${server.PORT}/empty.html`) + ]); + expect(serverRequest.headers.cookie).toBe('a=v; b=v'); + + const [serverRequest2] = await Promise.all([ + server.waitForRequest('/empty.html'), + request.get(`http://two.com:${server.PORT}/empty.html`) + ]); + expect(serverRequest2.headers.cookie).toBeFalsy(); +}); + +it('should do case-insensitive match of cookie domain', async ({ request, server }) => { + server.setRoute('/setcookie.html', (req, res) => { + res.setHeader('Set-Cookie', ['a=v; domain=One.com', 'b=v; domain=.B.oNe.com']); + res.end(); + }); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + const [serverRequest] = await Promise.all([ + server.waitForRequest('/empty.html'), + request.get(`http://www.b.one.com:${server.PORT}/empty.html`) + ]); + expect(serverRequest.headers.cookie).toBe('a=v; b=v'); +}); + +it('should do case-insensitive match of request domain', async ({ request, server }) => { + server.setRoute('/setcookie.html', (req, res) => { + res.setHeader('Set-Cookie', ['a=v; domain=one.com', 'b=v; domain=.b.one.com']); + res.end(); + }); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + const [serverRequest] = await Promise.all([ + server.waitForRequest('/empty.html'), + request.get(`http://WWW.B.ONE.COM:${server.PORT}/empty.html`) + ]); + expect(serverRequest.headers.cookie).toBe('a=v; b=v'); +}); + +it('should send secure cookie over https', async ({ request, server, httpsServer }) => { + server.setRoute('/setcookie.html', (req, res) => { + res.setHeader('Set-Cookie', ['a=v; secure', 'b=v']); + res.end(); + }); + await request.get(`${server.PREFIX}/setcookie.html`); + const [serverRequest] = await Promise.all([ + httpsServer.waitForRequest('/empty.html'), + request.get(httpsServer.EMPTY_PAGE) + ]); + expect(serverRequest.headers.cookie).toBe('a=v; b=v'); +}); + +it('should send not expired cookies', async ({ request, server }) => { + server.setRoute('/setcookie.html', (req, res) => { + const tomorrow = new Date(); + tomorrow.setDate(tomorrow.getDate() + 1); + res.setHeader('Set-Cookie', ['a=v', `b=v; expires=${tomorrow.toUTCString()}`]); + res.end(); + }); + await request.get(`${server.PREFIX}/setcookie.html`); + const [serverRequest] = await Promise.all([ + server.waitForRequest('/empty.html'), + request.get(server.EMPTY_PAGE) + ]); + expect(serverRequest.headers.cookie).toBe('a=v; b=v'); +}); + +it('should remove expired cookies', async ({ request, server }) => { + server.setRoute('/setcookie.html', (req, res) => { + res.setHeader('Set-Cookie', ['a=v', `b=v; expires=${new Date().toUTCString()}`]); + res.end(); + }); + await request.get(`${server.PREFIX}/setcookie.html`); + const [serverRequest] = await Promise.all([ + server.waitForRequest('/empty.html'), + request.get(server.EMPTY_PAGE) + ]); + expect(serverRequest.headers.cookie).toBe('a=v'); +}); +