Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

feat(fetch): store cookies between requests (#9221)

Browse source
This commit is contained in:
Yury Semikhatsky 2021-09-29 17:15:32 -07:00 committed by GitHub
parent 5633520f45
commit 2d428c8a4e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 320 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

133
src/server/cookieStore.ts Normal file
View file

@ -0,0 +1,133 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as types from './types';
class Cookie {
private _raw: types.NetworkCookie;
constructor(data: types.NetworkCookie) {
this._raw = data;
}
name(): string {
return this._raw.name;
}
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.4
matches(url: URL): boolean {
if (this._raw.secure && url.protocol !== 'https:')
return false;
if (!domainMatches(url.hostname, this._raw.domain))
return false;
if (!pathMatches(url.pathname, this._raw.path))
return false;
return true;
}
equals(other: Cookie) {
return this._raw.name === other._raw.name &&
this._raw.domain === other._raw.domain &&
this._raw.path === other._raw.path;
}
networkCookie(): types.NetworkCookie {
return this._raw;
}
updateExpiresFrom(other: Cookie) {
this._raw.expires = other._raw.expires;
}
expired() {
if (this._raw.expires === -1)
return false;
return this._raw.expires * 1000 < Date.now();
}
}
export class CookieStore {
private readonly _nameToCookies: Map<string, Set<Cookie>> = new Map();
addCookies(cookies: types.NetworkCookie[]) {
for (const cookie of cookies)
this._addCookie(new Cookie(cookie));
}
cookies(url: URL): types.NetworkCookie[] {
const result = [];
for (const cookie of this._allCookies()) {
if (cookie.matches(url))
result.push(cookie.networkCookie());
}
return result;
}
private _addCookie(cookie: Cookie) {
if (cookie.expired())
return;
let set = this._nameToCookies.get(cookie.name());
if (!set) {
set = new Set();
this._nameToCookies.set(cookie.name(), set);
}
CookieStore.pruneExpired(set);
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.3
for (const other of set) {
if (other.equals(cookie)) {
cookie.updateExpiresFrom(other);
set.delete(other);
}
}
set.add(cookie);
}
private *_allCookies(): IterableIterator<Cookie> {
for (const [name, cookies] of this._nameToCookies) {
CookieStore.pruneExpired(cookies);
for (const cookie of cookies)
yield cookie;
if (cookies.size === 0)
this._nameToCookies.delete(name);
}
}
private static pruneExpired(cookies: Set<Cookie>) {
for (const cookie of cookies) {
if (cookie.expired())
cookies.delete(cookie);
}
}
}
export function domainMatches(value: string, domain: string): boolean {
if (value === domain)
return true;
// Only strict match is allowed if domain doesn't start with '.' (host-only-flag is true in the spec)
if (!domain.startsWith('.'))
return false;
value = '.' + value;
return value.endsWith(domain);
}
function pathMatches(value: string, path: string): boolean {
if (value === path)
return true;
if (!value.endsWith('/'))
value = value + '/';
if (!path.endsWith('/'))
path = path + '/';
return value.startsWith(path);
}

46
src/server/fetch.ts
View file

@ -23,8 +23,9 @@ import zlib from 'zlib';
import { HTTPCredentials } from '../../types/types';
import { NameValue, NewRequestOptions } from '../common/types';
import { TimeoutSettings } from '../utils/timeoutSettings';
import { createGuid, getPlaywrightVersion, isFilePayload, monotonicTime } from '../utils/utils';
import { assert, createGuid, getPlaywrightVersion, isFilePayload, monotonicTime } from '../utils/utils';
import { BrowserContext } from './browserContext';
import { CookieStore, domainMatches } from './cookieStore';
import { MultipartFormData } from './formData';
import { SdkObject } from './instrumentation';
import { Playwright } from './playwright';
@ -73,8 +74,8 @@ export abstract class FetchRequest extends SdkObject {
abstract dispose(): void;
abstract _defaultOptions(): FetchRequestOptions;
abstract _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void>;
abstract _cookies(url: string): Promise<types.NetworkCookie[]>;
abstract _addCookies(cookies: types.NetworkCookie[]): Promise<void>;
abstract _cookies(url: URL): Promise<types.NetworkCookie[]>;
private _storeResponseBody(body: Buffer): string {
const uid = createGuid();
@ -155,15 +156,18 @@ export abstract class FetchRequest extends SdkObject {
const url = new URL(responseUrl);
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/');
const cookies: types.SetNetworkCookieParam[] = [];
const cookies: types.NetworkCookie[] = [];
for (const header of setCookie) {
// Decode cookie value?
const cookie: types.SetNetworkCookieParam | null = parseCookie(header);
const cookie: types.NetworkCookie | null = parseCookie(header);
if (!cookie)
continue;
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.3
if (!cookie.domain)
cookie.domain = url.hostname;
if (!canSetCookie(cookie.domain!, url.hostname))
else
assert(cookie.domain.startsWith('.'));
if (!domainMatches(url.hostname, cookie.domain!))
continue;
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4
if (!cookie.path || !cookie.path.startsWith('/'))
@ -177,7 +181,7 @@ export abstract class FetchRequest extends SdkObject {
private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) {
if (options.headers!['cookie'] !== undefined)
return;
const cookies = await this._cookies(url.toString());
const cookies = await this._cookies(url);
if (cookies.length) {
const valueArray = cookies.map(c => `${c.name}=${c.value}`);
options.headers!['cookie'] = valueArray.join('; ');
@ -326,17 +330,18 @@ export class BrowserContextFetchRequest extends FetchRequest {
};
}
async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void> {
async _addCookies(cookies: types.NetworkCookie[]): Promise<void> {
await this._context.addCookies(cookies);
}
async _cookies(url: string): Promise<types.NetworkCookie[]> {
return await this._context.cookies(url);
async _cookies(url: URL): Promise<types.NetworkCookie[]> {
return await this._context.cookies(url.toString());
}
}
export class GlobalFetchRequest extends FetchRequest {
private readonly _cookieStore: CookieStore = new CookieStore();
private readonly _options: FetchRequestOptions;
constructor(playwright: Playwright, options: Omit<NewRequestOptions, 'extraHTTPHeaders'> & { extraHTTPHeaders?: NameValue[] }) {
super(playwright);
@ -370,11 +375,12 @@ export class GlobalFetchRequest extends FetchRequest {
return this._options;
}
async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void> {
async _addCookies(cookies: types.NetworkCookie[]): Promise<void> {
this._cookieStore.addCookies(cookies);
}
async _cookies(url: string): Promise<types.NetworkCookie[]> {
return [];
async _cookies(url: URL): Promise<types.NetworkCookie[]> {
return this._cookieStore.cookies(url);
}
}
@ -387,15 +393,7 @@ function toHeadersArray(rawHeaders: string[]): types.HeadersArray {
const redirectStatus = [301, 302, 303, 307, 308];
function canSetCookie(cookieDomain: string, hostname: string) {
// TODO: check public suffix list?
hostname = '.' + hostname;
if (!cookieDomain.startsWith('.'))
cookieDomain = '.' + cookieDomain;
return hostname.endsWith(cookieDomain);
}
function parseCookie(header: string) {
function parseCookie(header: string): types.NetworkCookie | null {
const pairs = header.split(';').filter(s => s.trim().length > 0).map(p => p.split('=').map(s => s.trim()));
if (!pairs.length)
return null;
@ -424,7 +422,9 @@ function parseCookie(header: string) {
cookie.expires = Date.now() / 1000 + maxAgeSec;
break;
case 'domain':
cookie.domain = value || '';
cookie.domain = value.toLocaleLowerCase() || '';
if (cookie.domain && !cookie.domain.startsWith('.'))
cookie.domain = '.' + cookie.domain;
break;
case 'path':
cookie.path = value || '';

164
tests/global-fetch-cookie.spec.ts Normal file
View file

@ -0,0 +1,164 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import http from 'http';
import { FetchRequest } from '../index';
import { expect, playwrightTest } from './config/browserTest';
export type GlobalFetchFixtures = {
request: FetchRequest;
};
const it = playwrightTest.extend<GlobalFetchFixtures>({
request: async ({ playwright }, use) => {
const request = await playwright._newRequest({ ignoreHTTPSErrors: true });
await use(request);
await request.dispose();
},
});
it.skip(({ mode }) => mode !== 'default');
let prevAgent: http.Agent;
it.beforeAll(() => {
prevAgent = http.globalAgent;
http.globalAgent = new http.Agent({
// @ts-expect-error
lookup: (hostname, options, callback) => {
if (hostname === 'localhost' || hostname.endsWith('one.com') || hostname.endsWith('two.com'))
callback(null, '127.0.0.1', 4);
else
throw new Error(`Failed to resolve hostname: ${hostname}`);
}
});
});
it.afterAll(() => {
http.globalAgent = prevAgent;
});
it('should store cookie from Set-Cookie header', async ({ request, server }) => {
server.setRoute('/setcookie.html', (req, res) => {
res.setHeader('Set-Cookie', ['a=b', 'c=d; max-age=3600; domain=b.one.com; path=/input', 'e=f; domain=b.one.com; path=/input/subfolder']);
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
const [serverRequest] = await Promise.all([
server.waitForRequest('/input/button.html'),
request.get(`http://b.one.com:${server.PORT}/input/button.html`)
]);
expect(serverRequest.headers.cookie).toBe('c=d');
});
it('should filter outgoing cookies by path', async ({ request, server }) => {
server.setRoute('/setcookie.html', (req, res) => {
res.setHeader('Set-Cookie', ['a=v; path=/input/subfolder', 'b=v; path=/input', 'c=v;']);
res.end();
});
await request.get(`${server.PREFIX}/setcookie.html`);
const [serverRequest] = await Promise.all([
server.waitForRequest('/input/button.html'),
request.get(`${server.PREFIX}/input/button.html`)
]);
expect(serverRequest.headers.cookie).toBe('b=v; c=v');
});
it('should filter outgoing cookies by domain', async ({ request, server }) => {
server.setRoute('/setcookie.html', (req, res) => {
res.setHeader('Set-Cookie', ['a=v; domain=one.com', 'b=v; domain=.b.one.com', 'c=v; domain=other.com']);
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
const [serverRequest] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(`http://www.b.one.com:${server.PORT}/empty.html`)
]);
expect(serverRequest.headers.cookie).toBe('a=v; b=v');
const [serverRequest2] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(`http://two.com:${server.PORT}/empty.html`)
]);
expect(serverRequest2.headers.cookie).toBeFalsy();
});
it('should do case-insensitive match of cookie domain', async ({ request, server }) => {
server.setRoute('/setcookie.html', (req, res) => {
res.setHeader('Set-Cookie', ['a=v; domain=One.com', 'b=v; domain=.B.oNe.com']);
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
const [serverRequest] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(`http://www.b.one.com:${server.PORT}/empty.html`)
]);
expect(serverRequest.headers.cookie).toBe('a=v; b=v');
});
it('should do case-insensitive match of request domain', async ({ request, server }) => {
server.setRoute('/setcookie.html', (req, res) => {
res.setHeader('Set-Cookie', ['a=v; domain=one.com', 'b=v; domain=.b.one.com']);
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
const [serverRequest] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(`http://WWW.B.ONE.COM:${server.PORT}/empty.html`)
]);
expect(serverRequest.headers.cookie).toBe('a=v; b=v');
});
it('should send secure cookie over https', async ({ request, server, httpsServer }) => {
server.setRoute('/setcookie.html', (req, res) => {
res.setHeader('Set-Cookie', ['a=v; secure', 'b=v']);
res.end();
});
await request.get(`${server.PREFIX}/setcookie.html`);
const [serverRequest] = await Promise.all([
httpsServer.waitForRequest('/empty.html'),
request.get(httpsServer.EMPTY_PAGE)
]);
expect(serverRequest.headers.cookie).toBe('a=v; b=v');
});
it('should send not expired cookies', async ({ request, server }) => {
server.setRoute('/setcookie.html', (req, res) => {
const tomorrow = new Date();
tomorrow.setDate(tomorrow.getDate() + 1);
res.setHeader('Set-Cookie', ['a=v', `b=v; expires=${tomorrow.toUTCString()}`]);
res.end();
});
await request.get(`${server.PREFIX}/setcookie.html`);
const [serverRequest] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(server.EMPTY_PAGE)
]);
expect(serverRequest.headers.cookie).toBe('a=v; b=v');
});
it('should remove expired cookies', async ({ request, server }) => {
server.setRoute('/setcookie.html', (req, res) => {
res.setHeader('Set-Cookie', ['a=v', `b=v; expires=${new Date().toUTCString()}`]);
res.end();
});
await request.get(`${server.PREFIX}/setcookie.html`);
const [serverRequest] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(server.EMPTY_PAGE)
]);
expect(serverRequest.headers.cookie).toBe('a=v');
});