feature(navigation): implement networkilde0 and networkidle2 (#263)

This commit is contained in:
Dmitry Gozman 2019-12-16 16:32:04 -08:00 committed by Pavel Feldman
parent 6d404b0827
commit f9f7d5c55a
9 changed files with 153 additions and 95 deletions

View file

@ -23,7 +23,7 @@ import * as js from '../javascript';
import * as network from '../network';
import { CDPSession } from './Connection';
import { EVALUATION_SCRIPT_URL, ExecutionContextDelegate } from './ExecutionContext';
import { NetworkManager, NetworkManagerEvents } from './NetworkManager';
import { NetworkManager } from './NetworkManager';
import { Page } from '../page';
import { Protocol } from './protocol';
import { Events } from '../events';
@ -68,11 +68,6 @@ export class FrameManager extends EventEmitter implements PageDelegate {
(this._page as any).overrides = new Overrides(client);
(this._page as any).interception = new Interception(this._networkManager);
this._networkManager.on(NetworkManagerEvents.Request, event => this._page.emit(Events.Page.Request, event));
this._networkManager.on(NetworkManagerEvents.Response, event => this._page.emit(Events.Page.Response, event));
this._networkManager.on(NetworkManagerEvents.RequestFailed, event => this._page.emit(Events.Page.RequestFailed, event));
this._networkManager.on(NetworkManagerEvents.RequestFinished, event => this._page.emit(Events.Page.RequestFinished, event));
this._client.on('Inspector.targetCrashed', event => this._onTargetCrashed());
this._client.on('Log.entryAdded', event => this._onLogEntryAdded(event));
this._client.on('Page.fileChooserOpened', event => this._onFileChooserOpened(event));
@ -115,7 +110,6 @@ export class FrameManager extends EventEmitter implements PageDelegate {
}
async navigateFrame(frame: frames.Frame, url: string, options: frames.GotoOptions = {}): Promise<network.Response | null> {
assertNoLegacyNavigationOptions(options);
const {
referer = this._networkManager.extraHTTPHeaders()['referer'],
waitUntil = (['load'] as frames.LifecycleEvent[]),
@ -151,7 +145,6 @@ export class FrameManager extends EventEmitter implements PageDelegate {
}
async waitForFrameNavigation(frame: frames.Frame, options: frames.NavigateOptions = {}): Promise<network.Response | null> {
assertNoLegacyNavigationOptions(options);
const {
waitUntil = (['load'] as frames.LifecycleEvent[]),
timeout = this._page._timeoutSettings.navigationTimeout(),
@ -531,12 +524,6 @@ export class FrameManager extends EventEmitter implements PageDelegate {
}
}
function assertNoLegacyNavigationOptions(options: frames.NavigateOptions) {
assert((options as any)['networkIdleTimeout'] === undefined, 'ERROR: networkIdleTimeout option is no longer supported.');
assert((options as any)['networkIdleInflight'] === undefined, 'ERROR: networkIdleInflight option is no longer supported.');
assert((options as any).waitUntil !== 'networkidle', 'ERROR: "networkidle" option is no longer supported. Use "networkidle2" instead');
}
function toRemoteObject(handle: dom.ElementHandle): Protocol.Runtime.RemoteObject {
return handle._remoteObject as Protocol.Runtime.RemoteObject;
}

View file

@ -15,7 +15,6 @@
* limitations under the License.
*/
import { EventEmitter } from 'events';
import { CDPSession } from './Connection';
import { Page } from '../page';
import { assert, debugError, helper } from '../helper';
@ -23,14 +22,7 @@ import { Protocol } from './protocol';
import * as network from '../network';
import * as frames from '../frames';
export const NetworkManagerEvents = {
Request: Symbol('Events.NetworkManager.Request'),
Response: Symbol('Events.NetworkManager.Response'),
RequestFailed: Symbol('Events.NetworkManager.RequestFailed'),
RequestFinished: Symbol('Events.NetworkManager.RequestFinished'),
};
export class NetworkManager extends EventEmitter {
export class NetworkManager {
private _client: CDPSession;
private _ignoreHTTPSErrors: boolean;
private _page: Page;
@ -46,7 +38,6 @@ export class NetworkManager extends EventEmitter {
private _requestIdToInterceptionId = new Map<string, string>();
constructor(client: CDPSession, ignoreHTTPSErrors: boolean, page: Page) {
super();
this._client = client;
this._ignoreHTTPSErrors = ignoreHTTPSErrors;
this._page = page;
@ -203,7 +194,7 @@ export class NetworkManager extends EventEmitter {
const documentId = isNavigationRequest ? event.loaderId : undefined;
const request = new InterceptableRequest(this._client, frame, interceptionId, documentId, this._userRequestInterceptionEnabled, event, redirectChain);
this._requestIdToRequest.set(event.requestId, request);
this.emit(NetworkManagerEvents.Request, request.request);
this._page._frameManager.requestStarted(request.request);
}
_createResponse(request: InterceptableRequest, responsePayload: Protocol.Network.Response): network.Response {
@ -221,8 +212,8 @@ export class NetworkManager extends EventEmitter {
response._requestFinished(new Error('Response body is unavailable for redirect responses'));
this._requestIdToRequest.delete(request._requestId);
this._attemptedAuthentications.delete(request._interceptionId);
this.emit(NetworkManagerEvents.Response, response);
this.emit(NetworkManagerEvents.RequestFinished, request.request);
this._page._frameManager.requestReceivedResponse(response);
this._page._frameManager.requestFinished(request.request);
}
_onResponseReceived(event: Protocol.Network.responseReceivedPayload) {
@ -231,7 +222,7 @@ export class NetworkManager extends EventEmitter {
if (!request)
return;
const response = this._createResponse(request, event.response);
this.emit(NetworkManagerEvents.Response, response);
this._page._frameManager.requestReceivedResponse(response);
}
_onLoadingFinished(event: Protocol.Network.loadingFinishedPayload) {
@ -247,7 +238,7 @@ export class NetworkManager extends EventEmitter {
request.request.response()._requestFinished();
this._requestIdToRequest.delete(request._requestId);
this._attemptedAuthentications.delete(request._interceptionId);
this.emit(NetworkManagerEvents.RequestFinished, request.request);
this._page._frameManager.requestFinished(request.request);
}
_onLoadingFailed(event: Protocol.Network.loadingFailedPayload) {
@ -261,8 +252,8 @@ export class NetworkManager extends EventEmitter {
response._requestFinished();
this._requestIdToRequest.delete(request._requestId);
this._attemptedAuthentications.delete(request._interceptionId);
request.request._setFailureText(event.errorText, event.canceled);
this.emit(NetworkManagerEvents.RequestFailed, request.request);
request.request._setFailureText(event.errorText);
this._page._frameManager.requestFailed(request.request, event.canceled);
}
}

View file

@ -23,7 +23,7 @@ import * as dom from '../dom';
import { JugglerSession } from './Connection';
import { ExecutionContextDelegate } from './ExecutionContext';
import { Page, PageDelegate } from '../page';
import { NetworkManager, NetworkManagerEvents } from './NetworkManager';
import { NetworkManager } from './NetworkManager';
import { Events } from '../events';
import * as dialog from '../dialog';
import { Protocol } from './protocol';
@ -67,10 +67,6 @@ export class FrameManager extends EventEmitter implements PageDelegate {
helper.addEventListener(this._session, 'Page.dialogOpened', this._onDialogOpened.bind(this)),
helper.addEventListener(this._session, 'Page.bindingCalled', this._onBindingCalled.bind(this)),
helper.addEventListener(this._session, 'Page.fileChooserOpened', this._onFileChooserOpened.bind(this)),
helper.addEventListener(this._networkManager, NetworkManagerEvents.Request, request => this._page.emit(Events.Page.Request, request)),
helper.addEventListener(this._networkManager, NetworkManagerEvents.Response, response => this._page.emit(Events.Page.Response, response)),
helper.addEventListener(this._networkManager, NetworkManagerEvents.RequestFinished, request => this._page.emit(Events.Page.RequestFinished, request)),
helper.addEventListener(this._networkManager, NetworkManagerEvents.RequestFailed, request => this._page.emit(Events.Page.RequestFailed, request)),
];
(this._page as any).interception = new Interception(this._networkManager);
(this._page as any).accessibility = new Accessibility(session);

View file

@ -15,28 +15,19 @@
* limitations under the License.
*/
import { EventEmitter } from 'events';
import { assert, debugError, helper, RegisteredListener } from '../helper';
import { JugglerSession } from './Connection';
import { Page } from '../page';
import * as network from '../network';
import * as frames from '../frames';
export const NetworkManagerEvents = {
RequestFailed: Symbol('NetworkManagerEvents.RequestFailed'),
RequestFinished: Symbol('NetworkManagerEvents.RequestFinished'),
Request: Symbol('NetworkManagerEvents.Request'),
Response: Symbol('NetworkManagerEvents.Response'),
};
export class NetworkManager extends EventEmitter {
export class NetworkManager {
private _session: JugglerSession;
private _requests: Map<string, InterceptableRequest>;
private _page: Page;
private _eventListeners: RegisteredListener[];
constructor(session: JugglerSession, page: Page) {
super();
this._session = session;
this._requests = new Map();
@ -80,7 +71,7 @@ export class NetworkManager extends EventEmitter {
}
const request = new InterceptableRequest(this._session, frame, redirectChain, event);
this._requests.set(request._id, request);
this.emit(NetworkManagerEvents.Request, request.request);
this._page._frameManager.requestStarted(request.request);
}
_onResponseReceived(event) {
@ -100,7 +91,7 @@ export class NetworkManager extends EventEmitter {
for (const {name, value} of event.headers)
headers[name.toLowerCase()] = value;
const response = new network.Response(request.request, event.status, event.statusText, headers, remoteAddress, getResponseBody);
this.emit(NetworkManagerEvents.Response, response);
this._page._frameManager.requestReceivedResponse(response);
}
_onRequestFinished(event) {
@ -116,7 +107,7 @@ export class NetworkManager extends EventEmitter {
this._requests.delete(request._id);
response._requestFinished();
}
this.emit(NetworkManagerEvents.RequestFinished, request.request);
this._page._frameManager.requestFinished(request.request);
}
_onRequestFailed(event) {
@ -126,8 +117,8 @@ export class NetworkManager extends EventEmitter {
this._requests.delete(request._id);
if (request.request.response())
request.request.response()._requestFinished();
request.request._setFailureText(event.errorCode, event.errorCode === 'NS_BINDING_ABORTED');
this.emit(NetworkManagerEvents.RequestFailed, request.request);
request.request._setFailureText(event.errorCode);
this._page._frameManager.requestFailed(request.request, event.errorCode === 'NS_BINDING_ABORTED');
}
}

View file

@ -46,7 +46,8 @@ export type GotoOptions = NavigateOptions & {
referer?: string,
};
export type LifecycleEvent = 'load' | 'domcontentloaded';
export type LifecycleEvent = 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2';
const kLifecycleEvents: Set<LifecycleEvent> = new Set(['load', 'domcontentloaded', 'networkidle0', 'networkidle2']);
export type WaitForOptions = types.TimeoutOptions & { waitFor?: boolean };
@ -114,6 +115,12 @@ export class FrameManager {
for (const watcher of this._lifecycleWatchers)
watcher._onCommittedNewDocumentNavigation(frame);
}
this._stopNetworkIdleTimer(frame, 'networkidle0');
if (frame._inflightRequests === 0)
this._startNetworkIdleTimer(frame, 'networkidle0');
this._stopNetworkIdleTimer(frame, 'networkidle2');
if (frame._inflightRequests <= 2)
this._startNetworkIdleTimer(frame, 'networkidle2');
this._page.emit(Events.Page.FrameNavigated, frame);
}
@ -166,6 +173,42 @@ export class FrameManager {
this._page.emit(Events.Page.DOMContentLoaded);
}
requestStarted(request: network.Request) {
if (request.frame())
this._incrementRequestCount(request.frame());
if (request._documentId && request.frame() && !request.redirectChain().length) {
for (const watcher of this._lifecycleWatchers)
watcher._onNavigationRequest(request.frame(), request);
}
this._page.emit(Events.Page.Request, request);
}
requestReceivedResponse(response: network.Response) {
this._page.emit(Events.Page.Response, response);
}
requestFinished(request: network.Request) {
if (request.frame())
this._decrementRequestCount(request.frame());
this._page.emit(Events.Page.RequestFinished, request);
}
requestFailed(request: network.Request, canceled: boolean) {
if (request.frame())
this._decrementRequestCount(request.frame());
if (request._documentId && request.frame()) {
const isCurrentDocument = request.frame()._lastDocumentId === request._documentId;
if (!isCurrentDocument) {
let errorText = request.failure().errorText;
if (canceled)
errorText += '; maybe frame was detached?';
for (const watcher of this._lifecycleWatchers)
watcher._onAbortedNewDocumentNavigation(request.frame(), request._documentId, errorText);
}
}
this._page.emit(Events.Page.RequestFailed, request);
}
private _removeFramesRecursively(frame: Frame) {
for (const child of frame.childFrames())
this._removeFramesRecursively(child);
@ -175,6 +218,36 @@ export class FrameManager {
watcher._onFrameDetached(frame);
this._page.emit(Events.Page.FrameDetached, frame);
}
private _decrementRequestCount(frame: Frame) {
frame._inflightRequests--;
if (frame._inflightRequests === 0)
this._startNetworkIdleTimer(frame, 'networkidle0');
if (frame._inflightRequests === 2)
this._startNetworkIdleTimer(frame, 'networkidle2');
}
private _incrementRequestCount(frame: Frame) {
frame._inflightRequests++;
if (frame._inflightRequests === 1)
this._stopNetworkIdleTimer(frame, 'networkidle0');
if (frame._inflightRequests === 3)
this._stopNetworkIdleTimer(frame, 'networkidle2');
}
private _startNetworkIdleTimer(frame: Frame, event: LifecycleEvent) {
assert(!frame._networkIdleTimers.has(event));
if (frame._firedLifecycleEvents.has(event))
return;
frame._networkIdleTimers.set(event, setTimeout(() => {
this.frameLifecycleEvent(frame._id, event);
}, 500));
}
private _stopNetworkIdleTimer(frame: Frame, event: LifecycleEvent) {
clearTimeout(frame._networkIdleTimers.get(event));
frame._networkIdleTimers.delete(event);
}
}
export class Frame {
@ -188,6 +261,8 @@ export class Frame {
private _contextData = new Map<ContextType, ContextData>();
private _childFrames = new Set<Frame>();
_name: string;
_inflightRequests = 0;
readonly _networkIdleTimers = new Map<LifecycleEvent, NodeJS.Timer>();
constructor(page: Page, id: string, parentFrame: Frame | null) {
this._id = id;
@ -713,7 +788,7 @@ export class LifecycleWatcher {
waitUntil = waitUntil.slice();
else if (typeof waitUntil === 'string')
waitUntil = [waitUntil];
if (waitUntil.some(e => e !== 'load' && e !== 'domcontentloaded'))
if (waitUntil.some(e => !kLifecycleEvents.has(e)))
throw new Error('Unsupported waitUntil option');
this._expectedLifecycle = waitUntil.slice();
this._frame = frame;

View file

@ -101,24 +101,10 @@ export class Request {
this._headers = headers;
this._waitForResponsePromise = new Promise(f => this._waitForResponsePromiseCallback = f);
this._waitForFinishedPromise = new Promise(f => this._waitForFinishedPromiseCallback = f);
if (documentId && frame) {
for (const watcher of frame._page._frameManager._lifecycleWatchers)
watcher._onNavigationRequest(frame, this);
}
}
_setFailureText(failureText: string, canceled: boolean) {
_setFailureText(failureText: string) {
this._failureText = failureText;
if (this._documentId && this._frame) {
const isCurrentDocument = this._frame._lastDocumentId === this._documentId;
if (!isCurrentDocument) {
let errorText = failureText;
if (canceled)
errorText += '; maybe frame was detached?';
for (const watcher of this._frame._page._frameManager._lifecycleWatchers)
watcher._onAbortedNewDocumentNavigation(this._frame, this._documentId, errorText);
}
}
this._waitForFinishedPromiseCallback();
}

View file

@ -24,7 +24,7 @@ import * as network from '../network';
import { TargetSession } from './Connection';
import { Events } from '../events';
import { ExecutionContextDelegate, EVALUATION_SCRIPT_URL } from './ExecutionContext';
import { NetworkManager, NetworkManagerEvents } from './NetworkManager';
import { NetworkManager } from './NetworkManager';
import { Page, PageDelegate } from '../page';
import { Protocol } from './protocol';
import * as dialog from '../dialog';
@ -58,10 +58,6 @@ export class FrameManager extends EventEmitter implements PageDelegate {
this._isolatedWorlds = new Set();
this._page = new Page(this, browserContext);
this._networkManager = new NetworkManager(this._page);
this._networkManager.on(NetworkManagerEvents.Request, event => this._page.emit(Events.Page.Request, event));
this._networkManager.on(NetworkManagerEvents.Response, event => this._page.emit(Events.Page.Response, event));
this._networkManager.on(NetworkManagerEvents.RequestFailed, event => this._page.emit(Events.Page.RequestFailed, event));
this._networkManager.on(NetworkManagerEvents.RequestFinished, event => this._page.emit(Events.Page.RequestFinished, event));
}
setSession(session: TargetSession) {

View file

@ -15,7 +15,6 @@
* limitations under the License.
*/
import { EventEmitter } from 'events';
import { TargetSession } from './Connection';
import { Page } from '../page';
import { assert, helper, RegisteredListener } from '../helper';
@ -23,14 +22,7 @@ import { Protocol } from './protocol';
import * as network from '../network';
import * as frames from '../frames';
export const NetworkManagerEvents = {
Request: Symbol('Events.NetworkManager.Request'),
Response: Symbol('Events.NetworkManager.Response'),
RequestFailed: Symbol('Events.NetworkManager.RequestFailed'),
RequestFinished: Symbol('Events.NetworkManager.RequestFinished'),
};
export class NetworkManager extends EventEmitter {
export class NetworkManager {
private _session: TargetSession;
private _page: Page;
private _requestIdToRequest = new Map<string, InterceptableRequest>();
@ -40,7 +32,6 @@ export class NetworkManager extends EventEmitter {
private _sessionListeners: RegisteredListener[] = [];
constructor(page: Page) {
super();
this._page = page;
}
@ -103,7 +94,7 @@ export class NetworkManager extends EventEmitter {
const documentId = isNavigationRequest ? this._session._sessionId + '::' + event.loaderId : undefined;
const request = new InterceptableRequest(frame, undefined, event, redirectChain, documentId);
this._requestIdToRequest.set(event.requestId, request);
this.emit(NetworkManagerEvents.Request, request.request);
this._page._frameManager.requestStarted(request.request);
}
_createResponse(request: InterceptableRequest, responsePayload: Protocol.Network.Response): network.Response {
@ -121,8 +112,8 @@ export class NetworkManager extends EventEmitter {
response._requestFinished(new Error('Response body is unavailable for redirect responses'));
this._requestIdToRequest.delete(request._requestId);
this._attemptedAuthentications.delete(request._interceptionId);
this.emit(NetworkManagerEvents.Response, response);
this.emit(NetworkManagerEvents.RequestFinished, request.request);
this._page._frameManager.requestReceivedResponse(response);
this._page._frameManager.requestFinished(request.request);
}
_onResponseReceived(event: Protocol.Network.responseReceivedPayload) {
@ -131,7 +122,7 @@ export class NetworkManager extends EventEmitter {
if (!request)
return;
const response = this._createResponse(request, event.response);
this.emit(NetworkManagerEvents.Response, response);
this._page._frameManager.requestReceivedResponse(response);
}
_onLoadingFinished(event: Protocol.Network.loadingFinishedPayload) {
@ -147,7 +138,7 @@ export class NetworkManager extends EventEmitter {
request.request.response()._requestFinished();
this._requestIdToRequest.delete(request._requestId);
this._attemptedAuthentications.delete(request._interceptionId);
this.emit(NetworkManagerEvents.RequestFinished, request.request);
this._page._frameManager.requestFinished(request.request);
}
_onLoadingFailed(event: Protocol.Network.loadingFailedPayload) {
@ -161,8 +152,8 @@ export class NetworkManager extends EventEmitter {
response._requestFinished();
this._requestIdToRequest.delete(request._requestId);
this._attemptedAuthentications.delete(request._interceptionId);
request.request._setFailureText(event.errorText, event.errorText.includes('cancelled'));
this.emit(NetworkManagerEvents.RequestFailed, request.request);
request.request._setFailureText(event.errorText);
this._page._frameManager.requestFailed(request.request, event.errorText.includes('cancelled'));
}
}

View file

@ -16,6 +16,7 @@
*/
const utils = require('./utils');
const { performance } = require('perf_hooks');
module.exports.addTests = function({testRunner, expect, playwright, FFOX, CHROME, WEBKIT}) {
const {describe, xdescribe, fdescribe} = testRunner;
@ -132,11 +133,11 @@ module.exports.addTests = function({testRunner, expect, playwright, FFOX, CHROME
const response = await page.goto(server.PREFIX + '/grid.html');
expect(response.status()).toBe(200);
});
xit('should navigate to empty page with networkidle0', async({page, server}) => {
it('should navigate to empty page with networkidle0', async({page, server}) => {
const response = await page.goto(server.EMPTY_PAGE, {waitUntil: 'networkidle0'});
expect(response.status()).toBe(200);
});
xit('should navigate to empty page with networkidle2', async({page, server}) => {
it('should navigate to empty page with networkidle2', async({page, server}) => {
const response = await page.goto(server.EMPTY_PAGE, {waitUntil: 'networkidle2'});
expect(response.status()).toBe(200);
});
@ -166,10 +167,10 @@ module.exports.addTests = function({testRunner, expect, playwright, FFOX, CHROME
await page.goto(httpsServer.PREFIX + '/redirect/1.html').catch(e => error = e);
expectSSLError(error.message);
});
xit('should throw if networkidle is passed as an option', async({page, server}) => {
it('should throw if networkidle is passed as an option', async({page, server}) => {
let error = null;
await page.goto(server.EMPTY_PAGE, {waitUntil: 'networkidle'}).catch(err => error = err);
expect(error.message).toContain('"networkidle" option is no longer supported');
expect(error.message).toContain('Unsupported waitUntil option');
});
it('should fail when main resources failed to load', async({page, server}) => {
let error = null;
@ -250,7 +251,7 @@ module.exports.addTests = function({testRunner, expect, playwright, FFOX, CHROME
expect(response.ok()).toBe(true);
expect(response.url()).toBe(server.EMPTY_PAGE);
});
xit('should wait for network idle to succeed navigation', async({page, server}) => {
it('should wait for network idle to succeed navigation', async({page, server}) => {
let responses = [];
// Hold on to a bunch of requests without answering.
server.setRoute('/fetch-request-a.js', (req, res) => responses.push(res));
@ -303,10 +304,54 @@ module.exports.addTests = function({testRunner, expect, playwright, FFOX, CHROME
response.end(`File not found`);
}
const now = performance.now();
const response = await navigationPromise;
expect(performance.now() - now).not.toBeLessThan(499);
// Expect navigation to succeed.
expect(response.ok()).toBe(true);
});
it('should wait for networkidle2 to succeed navigation', async({page, server}) => {
let responses = [];
// Hold on to a bunch of requests without answering.
server.setRoute('/fetch-request-a.js', (req, res) => responses.push(res));
server.setRoute('/fetch-request-b.js', (req, res) => responses.push(res));
server.setRoute('/fetch-request-c.js', (req, res) => responses.push(res));
server.setRoute('/fetch-request-d.js', (req, res) => responses.push(res));
const initialFetchResourcesRequested = Promise.all([
server.waitForRequest('/fetch-request-a.js'),
]);
// Navigate to a page which loads immediately and then does a bunch of
// requests via javascript's fetch method.
const navigationPromise = page.goto(server.PREFIX + '/networkidle.html', {
waitUntil: 'networkidle2',
});
// Track when the navigation gets completed.
let navigationFinished = false;
navigationPromise.then(() => navigationFinished = true);
// Wait for the page's 'load' event.
await new Promise(fulfill => page.once('load', fulfill));
expect(navigationFinished).toBe(false);
// Wait for the initial three resources to be requested.
await initialFetchResourcesRequested;
// Expect navigation still to be not finished.
expect(navigationFinished).toBe(false);
// Respond to initial requests.
for (const response of responses) {
response.statusCode = 404;
response.end(`File not found`);
}
const now = performance.now();
const response = await navigationPromise;
expect(performance.now() - now).not.toBeLessThan(499);
// Expect navigation to succeed with two outstanding network requests.
expect(response.ok()).toBe(true);
});
it('should not leak listeners during navigation', async({page, server}) => {
let warning = null;
const warningHandler = w => warning = w;