From b0a78432475e11c4c013987154aff38a2c7a6947 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Tue, 24 Aug 2021 13:17:58 -0700 Subject: [PATCH] chore: migrate tracing ResourceSnapshot to follow har entry format (#8391) This will ease the migration of tracing to har. --- src/server/snapshot/snapshotRenderer.ts | 21 ++++++--- src/server/snapshot/snapshotServer.ts | 9 ++-- src/server/snapshot/snapshotTypes.ts | 31 ++++++++----- src/server/snapshot/snapshotter.ts | 29 +++++++----- src/server/supplements/har/har.ts | 4 ++ src/server/supplements/har/harTracer.ts | 3 ++ src/server/trace/recorder/tracing.ts | 2 +- src/utils/httpServer.ts | 1 + src/web/traceViewer/ui/modelUtil.ts | 2 +- .../traceViewer/ui/networkResourceDetails.tsx | 44 ++++++++++--------- tests/snapshotter.spec.ts | 2 +- tests/tracing.spec.ts | 6 +-- 12 files changed, 95 insertions(+), 59 deletions(-) diff --git a/src/server/snapshot/snapshotRenderer.ts b/src/server/snapshot/snapshotRenderer.ts index 1d80ff6235..8ffa0f7d46 100644 --- a/src/server/snapshot/snapshotRenderer.ts +++ b/src/server/snapshot/snapshotRenderer.ts @@ -96,11 +96,11 @@ export class SnapshotRenderer { // First try locating exact resource belonging to this frame. for (const resource of this._resources) { - if (resource.timestamp >= snapshot.timestamp) + if (resource._monotonicTime >= snapshot.timestamp) break; - if (resource.frameId !== snapshot.frameId) + if (resource._frameref !== snapshot.frameId) continue; - if (resource.url === url) { + if (resource.request.url === url) { result = resource; break; } @@ -109,9 +109,9 @@ export class SnapshotRenderer { if (!result) { // Then fall back to resource with this URL to account for memory cache. for (const resource of this._resources) { - if (resource.timestamp >= snapshot.timestamp) + if (resource._monotonicTime >= snapshot.timestamp) break; - if (resource.url === url) + if (resource.request.url === url) return resource; } } @@ -120,7 +120,16 @@ export class SnapshotRenderer { // Patch override if necessary. for (const o of snapshot.resourceOverrides) { if (url === o.url && o.sha1) { - result = { ...result, responseSha1: o.sha1 }; + result = { + ...result, + response: { + ...result.response, + content: { + ...result.response.content, + _sha1: o.sha1, + } + }, + }; break; } } diff --git a/src/server/snapshot/snapshotServer.ts b/src/server/snapshot/snapshotServer.ts index dfd3c8a5da..b73321b82a 100644 --- a/src/server/snapshot/snapshotServer.ts +++ b/src/server/snapshot/snapshotServer.ts @@ -172,18 +172,21 @@ export class SnapshotServer { if (!resource) return false; - const sha1 = resource.responseSha1; + const sha1 = resource.response.content._sha1; + if (!sha1) + return false; + try { const content = this._snapshotStorage.resourceContent(sha1); if (!content) return false; response.statusCode = 200; - let contentType = resource.contentType; + let contentType = resource.response.content.mimeType; const isTextEncoding = /^text\/|^application\/(javascript|json)/.test(contentType); if (isTextEncoding && !contentType.includes('charset')) contentType = `${contentType}; charset=utf-8`; response.setHeader('Content-Type', contentType); - for (const { name, value } of resource.responseHeaders) { + for (const { name, value } of resource.response.headers) { try { response.setHeader(name, value.split('\n')); } catch (e) { diff --git a/src/server/snapshot/snapshotTypes.ts b/src/server/snapshot/snapshotTypes.ts index cf5368fd08..5dcc526ed1 100644 --- a/src/server/snapshot/snapshotTypes.ts +++ b/src/server/snapshot/snapshotTypes.ts @@ -15,18 +15,25 @@ */ export type ResourceSnapshot = { - pageId: string, - frameId: string, - url: string, - type: string, - contentType: string, - responseHeaders: { name: string, value: string }[], - requestHeaders: { name: string, value: string }[], - method: string, - status: number, - requestSha1: string, - responseSha1: string, - timestamp: number, + _frameref: string, + request: { + url: string, + method: string, + headers: { name: string, value: string }[], + postData?: { + text: string, + _sha1?: string, + }, + }, + response: { + status: number, + headers: { name: string, value: string }[], + content: { + mimeType: string, + _sha1?: string, + }, + }, + _monotonicTime: number, }; export type NodeSnapshot = diff --git a/src/server/snapshot/snapshotter.ts b/src/server/snapshot/snapshotter.ts index 49985acb9e..5d45c05d21 100644 --- a/src/server/snapshot/snapshotter.ts +++ b/src/server/snapshot/snapshotter.ts @@ -205,18 +205,22 @@ export class Snapshotter { } const resource: ResourceSnapshot = { - pageId: response.frame()._page.guid, - frameId: response.frame().guid, - url, - type: response.request().resourceType(), - contentType, - responseHeaders: response.headers(), - requestHeaders, - method, - status, - requestSha1, - responseSha1, - timestamp: monotonicTime() + _frameref: response.frame().guid, + request: { + url, + method, + headers: requestHeaders, + postData: requestSha1 ? { text: '', _sha1: requestSha1 } : undefined, + }, + response: { + status, + headers: response.headers(), + content: { + mimeType: contentType, + _sha1: responseSha1, + }, + }, + _monotonicTime: monotonicTime() }; this._delegate.onResourceSnapshot(resource); } @@ -252,6 +256,7 @@ const kMimeToExtension: { [key: string]: string } = { 'image/jpeg': 'jpeg', 'image/png': 'png', 'image/tiff': 'tiff', + 'image/svg+xml': 'svg', 'text/css': 'css', 'text/csv': 'csv', 'text/html': 'html', diff --git a/src/server/supplements/har/har.ts b/src/server/supplements/har/har.ts index e538bf220b..d8d21a23a6 100644 --- a/src/server/supplements/har/har.ts +++ b/src/server/supplements/har/har.ts @@ -55,6 +55,8 @@ export type Entry = { timings: Timings; serverIPAddress?: string; connection?: string; + _frameref: string; + _monotonicTime: number; _serverPort?: number; _securityDetails?: SecurityDetails; }; @@ -109,6 +111,7 @@ export type PostData = { mimeType: string; params: Param[]; text: string; + _sha1?: string; }; export type Param = { @@ -124,6 +127,7 @@ export type Content = { mimeType: string; text?: string; encoding?: string; + _sha1?: string; }; export type Cache = { diff --git a/src/server/supplements/har/harTracer.ts b/src/server/supplements/har/harTracer.ts index 1ce25d8e30..5f1ddfc1e0 100644 --- a/src/server/supplements/har/harTracer.ts +++ b/src/server/supplements/har/harTracer.ts @@ -22,6 +22,7 @@ import * as network from '../../network'; import { Page } from '../../page'; import * as har from './har'; import * as types from '../../types'; +import { monotonicTime } from '../../../utils/utils'; const FALLBACK_HTTP_VERSION = 'HTTP/1.1'; @@ -128,6 +129,8 @@ export class HarTracer { const pageEntry = this._ensurePageEntry(page); const harEntry: har.Entry = { pageref: pageEntry.id, + _frameref: request.frame().guid, + _monotonicTime: monotonicTime(), startedDateTime: new Date(), time: -1, request: { diff --git a/src/server/trace/recorder/tracing.ts b/src/server/trace/recorder/tracing.ts index 14e39314bd..00e53a32ef 100644 --- a/src/server/trace/recorder/tracing.ts +++ b/src/server/trace/recorder/tracing.ts @@ -331,7 +331,7 @@ function visitSha1s(object: any, sha1s: Set) { } if (typeof object === 'object') { for (const key in object) { - if (key === 'sha1' || key.endsWith('Sha1')) { + if (key === 'sha1' || key === '_sha1' || key.endsWith('Sha1')) { const sha1 = object[key]; if (sha1) sha1s.add(sha1); diff --git a/src/utils/httpServer.ts b/src/utils/httpServer.ts index 123bab554b..7db6e286ba 100644 --- a/src/utils/httpServer.ts +++ b/src/utils/httpServer.ts @@ -97,6 +97,7 @@ const extensionToMime: { [key: string]: string } = { 'html': 'text/html', 'jpeg': 'image/jpeg', 'jpg': 'image/jpeg', + 'gif': 'image/gif', 'js': 'application/javascript', 'png': 'image/png', 'ttf': 'font/ttf', diff --git a/src/web/traceViewer/ui/modelUtil.ts b/src/web/traceViewer/ui/modelUtil.ts index f2c8a41274..ee0471dfa0 100644 --- a/src/web/traceViewer/ui/modelUtil.ts +++ b/src/web/traceViewer/ui/modelUtil.ts @@ -91,7 +91,7 @@ export function resourcesForAction(action: ActionTraceEvent): ResourceSnapshot[] const nextAction = next(action); result = context(action).resources.filter(resource => { - return resource.timestamp > action.metadata.startTime && (!nextAction || resource.timestamp < nextAction.metadata.startTime); + return resource._monotonicTime > action.metadata.startTime && (!nextAction || resource._monotonicTime < nextAction.metadata.startTime); }); (action as any)[resourcesSymbol] = result; return result; diff --git a/src/web/traceViewer/ui/networkResourceDetails.tsx b/src/web/traceViewer/ui/networkResourceDetails.tsx index 5c6c93fb1e..7474f44463 100644 --- a/src/web/traceViewer/ui/networkResourceDetails.tsx +++ b/src/web/traceViewer/ui/networkResourceDetails.tsx @@ -36,15 +36,19 @@ export const NetworkResourceDetails: React.FunctionComponent<{ React.useEffect(() => { const readResources = async () => { - if (resource.requestSha1) { - const response = await fetch(`/sha1/${resource.requestSha1}`); - const requestResource = await response.text(); - setRequestBody(requestResource); + if (resource.request.postData) { + if (resource.request.postData._sha1) { + const response = await fetch(`/sha1/${resource.request.postData}`); + const requestResource = await response.text(); + setRequestBody(requestResource); + } else { + setRequestBody(resource.request.postData.text); + } } - if (resource.responseSha1) { - const useBase64 = resource.contentType.includes('image'); - const response = await fetch(`/sha1/${resource.responseSha1}`); + if (resource.response.content._sha1) { + const useBase64 = resource.response.content.mimeType.includes('image'); + const response = await fetch(`/sha1/${resource.response.content._sha1}`); if (useBase64) { const blob = await response.blob(); const reader = new FileReader(); @@ -58,7 +62,7 @@ export const NetworkResourceDetails: React.FunctionComponent<{ }; readResources(); - }, [expanded, resource.responseSha1, resource.requestSha1, resource.contentType]); + }, [expanded, resource]); function formatBody(body: string | null, contentType: string): string { if (body === null) @@ -93,33 +97,33 @@ export const NetworkResourceDetails: React.FunctionComponent<{ return 'status-neutral'; } - const requestContentTypeHeader = resource.requestHeaders.find(q => q.name === 'Content-Type'); + const requestContentTypeHeader = resource.request.headers.find(q => q.name === 'Content-Type'); const requestContentType = requestContentTypeHeader ? requestContentTypeHeader.value : ''; - const resourceName = resource.url.substring(resource.url.lastIndexOf('/') + 1); + const resourceName = resource.request.url.substring(resource.request.url.lastIndexOf('/') + 1); return
setSelected(index)}> -
{resource.status}
-
{resource.method}
+
{resource.response.status}
+
{resource.request.method}
{resourceName}
-
{resource.type}
+
{resource.response.content.mimeType}
} body={
URL
-
{resource.url}
+
{resource.request.url}
Request Headers
-
{resource.requestHeaders.map(pair => `${pair.name}: ${pair.value}`).join('\n')}
+
{resource.request.headers.map(pair => `${pair.name}: ${pair.value}`).join('\n')}
Response Headers
-
{resource.responseHeaders.map(pair => `${pair.name}: ${pair.value}`).join('\n')}
- {resource.requestSha1 ?
Request Body
: ''} - {resource.requestSha1 ?
{formatBody(requestBody, requestContentType)}
: ''} +
{resource.response.headers.map(pair => `${pair.name}: ${pair.value}`).join('\n')}
+ {resource.request.postData ?
Request Body
: ''} + {resource.request.postData ?
{formatBody(requestBody, requestContentType)}
: ''}
Response Body
- {!resource.responseSha1 ?
Response body is not available for this request.
: ''} + {!resource.response.content._sha1 ?
Response body is not available for this request.
: ''} {responseBody !== null && responseBody.dataUrl ? : ''} - {responseBody !== null && responseBody.text ?
{formatBody(responseBody.text, resource.contentType)}
: ''} + {responseBody !== null && responseBody.text ?
{formatBody(responseBody.text, resource.response.content.mimeType)}
: ''}
}/> ; diff --git a/tests/snapshotter.spec.ts b/tests/snapshotter.spec.ts index bf7e9db4a5..faad50d8c2 100644 --- a/tests/snapshotter.spec.ts +++ b/tests/snapshotter.spec.ts @@ -146,7 +146,7 @@ it.describe('snapshots', () => { await page.evaluate(() => { (document.styleSheets[0].cssRules[0] as any).style.color = 'blue'; }); const snapshot2 = await snapshotter.captureSnapshot(toImpl(page), 'snapshot1'); const resource = snapshot2.resourceByUrl(`http://localhost:${server.PORT}/style.css`); - expect(snapshotter.resourceContent(resource.responseSha1).toString()).toBe('button { color: blue; }'); + expect(snapshotter.resourceContent(resource.response.content._sha1).toString()).toBe('button { color: blue; }'); }); it('should capture iframe', async ({ page, server, toImpl, browserName, snapshotter, showSnapshot }) => { diff --git a/tests/tracing.spec.ts b/tests/tracing.spec.ts index 34f2a15736..8d1c9f555f 100644 --- a/tests/tracing.spec.ts +++ b/tests/tracing.spec.ts @@ -37,8 +37,8 @@ test('should collect trace with resources, but no js', async ({ context, page, s expect(events.find(e => e.metadata?.apiName === 'page.close')).toBeTruthy(); expect(events.some(e => e.type === 'frame-snapshot')).toBeTruthy(); - expect(events.some(e => e.type === 'resource-snapshot' && e.snapshot.url.endsWith('style.css'))).toBeTruthy(); - expect(events.some(e => e.type === 'resource-snapshot' && e.snapshot.url.endsWith('script.js'))).toBeFalsy(); + expect(events.some(e => e.type === 'resource-snapshot' && e.snapshot.request.url.endsWith('style.css'))).toBeTruthy(); + expect(events.some(e => e.type === 'resource-snapshot' && e.snapshot.request.url.endsWith('script.js'))).toBeFalsy(); expect(events.some(e => e.type === 'screencast-frame')).toBeTruthy(); }); @@ -239,7 +239,7 @@ test('should reset and export', async ({ context, page, server }, testInfo) => { expect(trace1.events.find(e => e.metadata?.apiName === 'page.hover')).toBeFalsy(); expect(trace1.events.find(e => e.metadata?.apiName === 'page.click' && e.metadata?.error?.error?.message === 'Action was interrupted')).toBeTruthy(); expect(trace1.events.some(e => e.type === 'frame-snapshot')).toBeTruthy(); - expect(trace1.events.some(e => e.type === 'resource-snapshot' && e.snapshot.url.endsWith('style.css'))).toBeTruthy(); + expect(trace1.events.some(e => e.type === 'resource-snapshot' && e.snapshot.request.url.endsWith('style.css'))).toBeTruthy(); const trace2 = await parseTrace(testInfo.outputPath('trace2.zip')); expect(trace2.events[0].type).toBe('context-options');