Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore(tracing): remove resource id (#8131)

Browse source
This commit is contained in:
Pavel Feldman 2021-08-10 21:23:31 -07:00 committed by GitHub
parent 75dfc15e62
commit 21b510c6e7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 66 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
src/server/snapshot/snapshotRenderer.ts
View file

@ -21,11 +21,13 @@ export class SnapshotRenderer {
private _index: number; private _index: number;
readonly snapshotName: string | undefined; readonly snapshotName: string | undefined;
private _resources: ResourceSnapshot[]; private _resources: ResourceSnapshot[];
private _snapshot: FrameSnapshot;
constructor(resources: ResourceSnapshot[], snapshots: FrameSnapshot[], index: number) { constructor(resources: ResourceSnapshot[], snapshots: FrameSnapshot[], index: number) {
this._resources = resources; this._resources = resources;
this._snapshots = snapshots; this._snapshots = snapshots;
this._index = index; this._index = index;
this._snapshot = snapshots[index];
this.snapshotName = snapshots[index].snapshotName; this.snapshotName = snapshots[index].snapshotName;
} }
@ -73,10 +75,10 @@ export class SnapshotRenderer {
return (n as any)._string; return (n as any)._string;
}; };
const snapshot = this._snapshots[this._index]; const snapshot = this._snapshot;
let html = visit(snapshot.html, this._index); let html = visit(snapshot.html, this._index);
if (!html) if (!html)
return { html: '', resources: {} }; return { html: '', pageId: snapshot.pageId, frameId: snapshot.frameId, index: this._index };
if (snapshot.doctype) if (snapshot.doctype)
html = `<!DOCTYPE ${snapshot.doctype}>` + html; html = `<!DOCTYPE ${snapshot.doctype}>` + html;
@ -85,26 +87,46 @@ export class SnapshotRenderer {
<script>${snapshotScript()}</script> <script>${snapshotScript()}</script>
`; `;
const resources: { [key: string]: { resourceId: string, sha1?: string } } = {}; return { html, pageId: snapshot.pageId, frameId: snapshot.frameId, index: this._index };
// First capture all resources for all frames, to account for memory cache. }
for (const resource of this._resources) {
if (resource.timestamp >= snapshot.timestamp) resourceByUrl(url: string): ResourceSnapshot | undefined {
break; const snapshot = this._snapshot;
resources[resource.url] = { resourceId: resource.resourceId }; let result: ResourceSnapshot | undefined;
}
// Then overwrite with the ones from our frame. // First try locating exact resource belonging to this frame.
for (const resource of this._resources) { for (const resource of this._resources) {
if (resource.timestamp >= snapshot.timestamp) if (resource.timestamp >= snapshot.timestamp)
break; break;
if (resource.frameId !== snapshot.frameId) if (resource.frameId !== snapshot.frameId)
continue; continue;
resources[resource.url] = { resourceId: resource.resourceId }; if (resource.url === url) {
result = resource;
break;
}
} }
for (const o of snapshot.resourceOverrides) {
const resource = resources[o.url]; if (!result) {
resource.sha1 = o.sha1; // Then fall back to resource with this URL to account for memory cache.
for (const resource of this._resources) {
if (resource.timestamp >= snapshot.timestamp)
break;
if (resource.url === url)
return resource;
}
} }
return { html, resources };
if (result) {
// Patch override if necessary.
for (const o of snapshot.resourceOverrides) {
if (url === o.url && o.sha1) {
result = { ...result, responseSha1: o.sha1 };
break;
}
}
}
return result;
} }
} }

57
src/server/snapshot/snapshotServer.ts
View file

@ -62,7 +62,7 @@ export class SnapshotServer {
private _serveServiceWorker(request: http.IncomingMessage, response: http.ServerResponse): boolean { private _serveServiceWorker(request: http.IncomingMessage, response: http.ServerResponse): boolean {
function serviceWorkerMain(self: any /* ServiceWorkerGlobalScope */) { function serviceWorkerMain(self: any /* ServiceWorkerGlobalScope */) {
const snapshotResources = new Map<string, { [key: string]: { resourceId?: string, sha1?: string } }>(); const snapshotIds = new Map<string, { frameId: string, index: number }>();
self.addEventListener('install', function(event: any) { self.addEventListener('install', function(event: any) {
}); });
@ -71,10 +71,6 @@ export class SnapshotServer {
event.waitUntil(self.clients.claim()); event.waitUntil(self.clients.claim());
}); });
function respond404(): Response {
return new Response(null, { status: 404 });
}
function respondNotAvailable(): Response { function respondNotAvailable(): Response {
return new Response('<body style="background: #ddd"></body>', { status: 200, headers: { 'Content-Type': 'text/html' } }); return new Response('<body style="background: #ddd"></body>', { status: 200, headers: { 'Content-Type': 'text/html' } });
} }
@ -100,34 +96,26 @@ export class SnapshotServer {
if (request.mode === 'navigate') { if (request.mode === 'navigate') {
const htmlResponse = await fetch(event.request); const htmlResponse = await fetch(event.request);
const { html, resources }: RenderedFrameSnapshot = await htmlResponse.json(); const { html, frameId, index }: RenderedFrameSnapshot = await htmlResponse.json();
if (!html) if (!html)
return respondNotAvailable(); return respondNotAvailable();
snapshotResources.set(snapshotUrl, resources); snapshotIds.set(snapshotUrl, { frameId, index });
const response = new Response(html, { status: 200, headers: { 'Content-Type': 'text/html' } }); const response = new Response(html, { status: 200, headers: { 'Content-Type': 'text/html' } });
return response; return response;
} }
const resources = snapshotResources.get(snapshotUrl)!; const { frameId, index } = snapshotIds.get(snapshotUrl)!;
const urlWithoutHash = removeHash(request.url); const url = removeHash(request.url);
const resource = resources[urlWithoutHash]; const complexUrl = btoa(JSON.stringify({ frameId, index, url }));
if (!resource) const fetchUrl = `/resources/${complexUrl}`;
return respond404();
const fetchUrl = resource.sha1 ?
`/resources/${resource.resourceId}/override/${resource.sha1}` :
`/resources/${resource.resourceId}`;
const fetchedResponse = await fetch(fetchUrl); const fetchedResponse = await fetch(fetchUrl);
const headers = new Headers(fetchedResponse.headers);
// We make a copy of the response, instead of just forwarding, // We make a copy of the response, instead of just forwarding,
// so that response url is not inherited as "/resources/...", but instead // so that response url is not inherited as "/resources/...", but instead
// as the original request url. // as the original request url.
// Response url turns into resource base uri that is used to resolve // Response url turns into resource base uri that is used to resolve
// relative links, e.g. url(/foo/bar) in style sheets. // relative links, e.g. url(/foo/bar) in style sheets.
if (resource.sha1) { const headers = new Headers(fetchedResponse.headers);
// No cache, so that we refetch overridden resources.
headers.set('Cache-Control', 'no-cache');
}
const response = new Response(fetchedResponse.body, { const response = new Response(fetchedResponse.body, {
status: fetchedResponse.status, status: fetchedResponse.status,
statusText: fetchedResponse.statusText, statusText: fetchedResponse.statusText,
@ -178,32 +166,13 @@ export class SnapshotServer {
} }
private _serveResource(request: http.IncomingMessage, response: http.ServerResponse): boolean { private _serveResource(request: http.IncomingMessage, response: http.ServerResponse): boolean {
// - /resources/<resourceId> const { frameId, index, url } = JSON.parse(Buffer.from(request.url!.substring('/resources/'.length), 'base64').toString());
// - /resources/<resourceId>/override/<overrideSha1> const snapshot = this._snapshotStorage.snapshotByIndex(frameId, index);
const parts = request.url!.split('/'); const resource = snapshot?.resourceByUrl(url);
if (!parts[0])
parts.shift();
if (!parts[parts.length - 1])
parts.pop();
if (parts[0] !== 'resources')
return false;
let resourceId;
let overrideSha1;
if (parts.length === 2) {
resourceId = parts[1];
} else if (parts.length === 4 && parts[2] === 'override') {
resourceId = parts[1];
overrideSha1 = parts[3];
} else {
return false;
}
const resource = this._snapshotStorage.resourceById(resourceId);
if (!resource) if (!resource)
return false; return false;
const sha1 = overrideSha1 || resource.responseSha1; const sha1 = resource.responseSha1;
try { try {
const content = this._snapshotStorage.resourceContent(sha1); const content = this._snapshotStorage.resourceContent(sha1);
if (!content) if (!content)

15
src/server/snapshot/snapshotStorage.ts
View file

@ -21,13 +21,12 @@ import { SnapshotRenderer } from './snapshotRenderer';
export interface SnapshotStorage { export interface SnapshotStorage {
resources(): ResourceSnapshot[]; resources(): ResourceSnapshot[];
resourceContent(sha1: string): Buffer | undefined; resourceContent(sha1: string): Buffer | undefined;
resourceById(resourceId: string): ResourceSnapshot | undefined;
snapshotByName(pageOrFrameId: string, snapshotName: string): SnapshotRenderer | undefined; snapshotByName(pageOrFrameId: string, snapshotName: string): SnapshotRenderer | undefined;
snapshotByIndex(frameId: string, index: number): SnapshotRenderer | undefined;
} }
export abstract class BaseSnapshotStorage extends EventEmitter implements SnapshotStorage { export abstract class BaseSnapshotStorage extends EventEmitter implements SnapshotStorage {
protected _resources: ResourceSnapshot[] = []; protected _resources: ResourceSnapshot[] = [];
protected _resourceMap = new Map<string, ResourceSnapshot>();
protected _frameSnapshots = new Map<string, { protected _frameSnapshots = new Map<string, {
raw: FrameSnapshot[], raw: FrameSnapshot[],
renderer: SnapshotRenderer[] renderer: SnapshotRenderer[]
@ -35,12 +34,10 @@ export abstract class BaseSnapshotStorage extends EventEmitter implements Snapsh
clear() { clear() {
this._resources = []; this._resources = [];
this._resourceMap.clear();
this._frameSnapshots.clear(); this._frameSnapshots.clear();
} }
addResource(resource: ResourceSnapshot): void { addResource(resource: ResourceSnapshot): void {
this._resourceMap.set(resource.resourceId, resource);
this._resources.push(resource); this._resources.push(resource);
} }
@ -63,10 +60,6 @@ export abstract class BaseSnapshotStorage extends EventEmitter implements Snapsh
abstract resourceContent(sha1: string): Buffer | undefined; abstract resourceContent(sha1: string): Buffer | undefined;
resourceById(resourceId: string): ResourceSnapshot | undefined {
return this._resourceMap.get(resourceId)!;
}
resources(): ResourceSnapshot[] { resources(): ResourceSnapshot[] {
return this._resources.slice(); return this._resources.slice();
} }
@ -75,4 +68,10 @@ export abstract class BaseSnapshotStorage extends EventEmitter implements Snapsh
const snapshot = this._frameSnapshots.get(pageOrFrameId); const snapshot = this._frameSnapshots.get(pageOrFrameId);
return snapshot?.renderer.find(r => r.snapshotName === snapshotName); return snapshot?.renderer.find(r => r.snapshotName === snapshotName);
} }
snapshotByIndex(frameId: string, index: number): SnapshotRenderer | undefined {
const snapshot = this._frameSnapshots.get(frameId);
return snapshot?.renderer[index];
}
} }

5
src/server/snapshot/snapshotTypes.ts
View file

@ -15,7 +15,6 @@
*/ */
export type ResourceSnapshot = { export type ResourceSnapshot = {
resourceId: string,
pageId: string, pageId: string,
frameId: string, frameId: string,
url: string, url: string,
@ -65,5 +64,7 @@ export type FrameSnapshot = {
export type RenderedFrameSnapshot = { export type RenderedFrameSnapshot = {
html: string; html: string;
resources: { [key: string]: { resourceId: string, sha1?: string } }; pageId: string;
frameId: string;
index: number;
}; };

1
src/server/snapshot/snapshotter.ts
View file

@ -207,7 +207,6 @@ export class Snapshotter {
const resource: ResourceSnapshot = { const resource: ResourceSnapshot = {
pageId: response.frame()._page.guid, pageId: response.frame()._page.guid,
frameId: response.frame().guid, frameId: response.frame().guid,
resourceId: response.guid,
url, url,
type: response.request().resourceType(), type: response.request().resourceType(),
contentType, contentType,

4
src/server/trace/viewer/traceViewer.ts
View file

@ -49,11 +49,11 @@ export class TraceViewer {
// - "/sha1/<sha1>" - trace resource bodies, used by network previews. // - "/sha1/<sha1>" - trace resource bodies, used by network previews.
// //
// Served by SnapshotServer // Served by SnapshotServer
// - "/resources/<resourceId>" - network resources from the trace. // - "/resources/" - network resources from the trace.
// - "/snapshot/" - root for snapshot frame. // - "/snapshot/" - root for snapshot frame.
// - "/snapshot/pageId/..." - actual snapshot html. // - "/snapshot/pageId/..." - actual snapshot html.
// - "/snapshot/service-worker.js" - service worker that intercepts snapshot resources // - "/snapshot/service-worker.js" - service worker that intercepts snapshot resources
// and translates them into "/resources/<resourceId>". // and translates them into network requests.
const actionTraces = fs.readdirSync(tracesDir).filter(name => name.endsWith('.trace')); const actionTraces = fs.readdirSync(tracesDir).filter(name => name.endsWith('.trace'));
const debugNames = actionTraces.map(name => { const debugNames = actionTraces.map(name => {
const tracePrefix = path.join(tracesDir, name.substring(0, name.indexOf('.trace'))); const tracePrefix = path.join(tracesDir, name.substring(0, name.indexOf('.trace')));

11
tests/snapshotter.spec.ts
View file

@ -62,9 +62,8 @@ it.describe('snapshots', () => {
}); });
await page.setContent('<link rel="stylesheet" href="style.css"><button>Hello</button>'); await page.setContent('<link rel="stylesheet" href="style.css"><button>Hello</button>');
const snapshot = await snapshotter.captureSnapshot(toImpl(page), 'snapshot'); const snapshot = await snapshotter.captureSnapshot(toImpl(page), 'snapshot');
const { resources } = snapshot.render(); const resource = snapshot.resourceByUrl(`http://localhost:${server.PORT}/style.css`);
const cssHref = `http://localhost:${server.PORT}/style.css`; expect(resource).toBeTruthy();
expect(resources[cssHref]).toBeTruthy();
}); });
it('should collect multiple', async ({ page, toImpl, snapshotter }) => { it('should collect multiple', async ({ page, toImpl, snapshotter }) => {
@ -126,10 +125,8 @@ it.describe('snapshots', () => {
await page.evaluate(() => { (document.styleSheets[0].cssRules[0] as any).style.color = 'blue'; }); await page.evaluate(() => { (document.styleSheets[0].cssRules[0] as any).style.color = 'blue'; });
const snapshot2 = await snapshotter.captureSnapshot(toImpl(page), 'snapshot1'); const snapshot2 = await snapshotter.captureSnapshot(toImpl(page), 'snapshot1');
const { resources } = snapshot2.render(); const resource = snapshot2.resourceByUrl(`http://localhost:${server.PORT}/style.css`);
const cssHref = `http://localhost:${server.PORT}/style.css`; expect(snapshotter.resourceContent(resource.responseSha1).toString()).toBe('button { color: blue; }');
const { sha1 } = resources[cssHref];
expect(snapshotter.resourceContent(sha1).toString()).toBe('button { color: blue; }');
}); });
it('should capture iframe', async ({ page, contextFactory, server, toImpl, browserName, snapshotter, snapshotPort }) => { it('should capture iframe', async ({ page, contextFactory, server, toImpl, browserName, snapshotter, snapshotPort }) => {