chore: migrate tracing ResourceSnapshot to follow har entry format (#8391)

This will ease the migration of tracing to har.

src/server/snapshot/snapshotRenderer.ts

@@ -96,11 +96,11 @@ export class SnapshotRenderer {
 
     // First try locating exact resource belonging to this frame.
     for (const resource of this._resources) {
-      if (resource.timestamp >= snapshot.timestamp)
+      if (resource._monotonicTime >= snapshot.timestamp)
         break;
-      if (resource.frameId !== snapshot.frameId)
+      if (resource._frameref !== snapshot.frameId)
         continue;
-      if (resource.url === url) {
+      if (resource.request.url === url) {
         result = resource;
         break;
       }
@@ -109,9 +109,9 @@ export class SnapshotRenderer {
     if (!result) {
       // Then fall back to resource with this URL to account for memory cache.
       for (const resource of this._resources) {
-        if (resource.timestamp >= snapshot.timestamp)
+        if (resource._monotonicTime >= snapshot.timestamp)
           break;
-        if (resource.url === url)
+        if (resource.request.url === url)
           return resource;
       }
     }
@@ -120,7 +120,16 @@ export class SnapshotRenderer {
       // Patch override if necessary.
       for (const o of snapshot.resourceOverrides) {
         if (url === o.url && o.sha1) {
-          result = { ...result, responseSha1: o.sha1 };
+          result = {
+            ...result,
+            response: {
+              ...result.response,
+              content: {
+                ...result.response.content,
+                _sha1: o.sha1,
+              }
+            },
+          };
           break;
         }
       }

src/server/snapshot/snapshotServer.ts

@@ -172,18 +172,21 @@ export class SnapshotServer {
     if (!resource)
       return false;
 
-    const sha1 = resource.responseSha1;
+    const sha1 = resource.response.content._sha1;
+    if (!sha1)
+      return false;
+
     try {
       const content = this._snapshotStorage.resourceContent(sha1);
       if (!content)
         return false;
       response.statusCode = 200;
-      let contentType = resource.contentType;
+      let contentType = resource.response.content.mimeType;
       const isTextEncoding = /^text\/|^application\/(javascript|json)/.test(contentType);
       if (isTextEncoding && !contentType.includes('charset'))
         contentType = `${contentType}; charset=utf-8`;
       response.setHeader('Content-Type', contentType);
-      for (const { name, value } of resource.responseHeaders) {
+      for (const { name, value } of resource.response.headers) {
         try {
           response.setHeader(name, value.split('\n'));
         } catch (e) {

src/server/snapshot/snapshotTypes.ts

@@ -15,18 +15,25 @@
  */
 
 export type ResourceSnapshot = {
-  pageId: string,
+  _frameref: string,
-  frameId: string,
+  request: {
-  url: string,
+    url: string,
-  type: string,
+    method: string,
-  contentType: string,
+    headers: { name: string, value: string }[],
-  responseHeaders: { name: string, value: string }[],
+    postData?: {
-  requestHeaders: { name: string, value: string }[],
+      text: string,
-  method: string,
+      _sha1?: string,
-  status: number,
+    },
-  requestSha1: string,
+  },
-  responseSha1: string,
+  response: {
-  timestamp: number,
+    status: number,
+    headers: { name: string, value: string }[],
+    content: {
+      mimeType: string,
+      _sha1?: string,
+    },
+  },
+  _monotonicTime: number,
 };
 
 export type NodeSnapshot =

src/server/snapshot/snapshotter.ts

@@ -205,18 +205,22 @@ export class Snapshotter {
     }
 
     const resource: ResourceSnapshot = {
-      pageId: response.frame()._page.guid,
+      _frameref: response.frame().guid,
-      frameId: response.frame().guid,
+      request: {
-      url,
+        url,
-      type: response.request().resourceType(),
+        method,
-      contentType,
+        headers: requestHeaders,
-      responseHeaders: response.headers(),
+        postData: requestSha1 ? { text: '', _sha1: requestSha1 } : undefined,
-      requestHeaders,
+      },
-      method,
+      response: {
-      status,
+        status,
-      requestSha1,
+        headers: response.headers(),
-      responseSha1,
+        content: {
-      timestamp: monotonicTime()
+          mimeType: contentType,
+          _sha1: responseSha1,
+        },
+      },
+      _monotonicTime: monotonicTime()
     };
     this._delegate.onResourceSnapshot(resource);
   }
@@ -252,6 +256,7 @@ const kMimeToExtension: { [key: string]: string } = {
   'image/jpeg': 'jpeg',
   'image/png': 'png',
   'image/tiff': 'tiff',
+  'image/svg+xml': 'svg',
   'text/css': 'css',
   'text/csv': 'csv',
   'text/html': 'html',

src/server/supplements/har/har.ts

@@ -55,6 +55,8 @@ export type Entry = {
   timings: Timings;
   serverIPAddress?: string;
   connection?: string;
+  _frameref: string;
+  _monotonicTime: number;
   _serverPort?: number;
   _securityDetails?: SecurityDetails;
 };
@@ -109,6 +111,7 @@ export type PostData = {
   mimeType: string;
   params: Param[];
   text: string;
+  _sha1?: string;
 };
 
 export type Param = {
@@ -124,6 +127,7 @@ export type Content = {
   mimeType: string;
   text?: string;
   encoding?: string;
+  _sha1?: string;
 };
 
 export type Cache = {

src/server/supplements/har/harTracer.ts

@@ -22,6 +22,7 @@ import * as network from '../../network';
 import { Page } from '../../page';
 import * as har from './har';
 import * as types from '../../types';
+import { monotonicTime } from '../../../utils/utils';
 
 const FALLBACK_HTTP_VERSION = 'HTTP/1.1';
 
@@ -128,6 +129,8 @@ export class HarTracer {
     const pageEntry = this._ensurePageEntry(page);
     const harEntry: har.Entry = {
       pageref: pageEntry.id,
+      _frameref: request.frame().guid,
+      _monotonicTime: monotonicTime(),
       startedDateTime: new Date(),
       time: -1,
       request: {

src/server/trace/recorder/tracing.ts

@@ -331,7 +331,7 @@ function visitSha1s(object: any, sha1s: Set<string>) {
   }
   if (typeof object === 'object') {
     for (const key in object) {
-      if (key === 'sha1' || key.endsWith('Sha1')) {
+      if (key === 'sha1' || key === '_sha1' || key.endsWith('Sha1')) {
         const sha1 = object[key];
         if (sha1)
           sha1s.add(sha1);

src/utils/httpServer.ts

@@ -97,6 +97,7 @@ const extensionToMime: { [key: string]: string } = {
   'html': 'text/html',
   'jpeg': 'image/jpeg',
   'jpg': 'image/jpeg',
+  'gif': 'image/gif',
   'js': 'application/javascript',
   'png': 'image/png',
   'ttf': 'font/ttf',

src/web/traceViewer/ui/modelUtil.ts

@@ -91,7 +91,7 @@ export function resourcesForAction(action: ActionTraceEvent): ResourceSnapshot[]
 
   const nextAction = next(action);
   result = context(action).resources.filter(resource => {
-    return resource.timestamp > action.metadata.startTime && (!nextAction || resource.timestamp < nextAction.metadata.startTime);
+    return resource._monotonicTime > action.metadata.startTime && (!nextAction || resource._monotonicTime < nextAction.metadata.startTime);
   });
   (action as any)[resourcesSymbol] = result;
   return result;

src/web/traceViewer/ui/networkResourceDetails.tsx

@@ -36,15 +36,19 @@ export const NetworkResourceDetails: React.FunctionComponent<{
 
   React.useEffect(() => {
     const readResources = async  () => {
-      if (resource.requestSha1) {
+      if (resource.request.postData) {
-        const response = await fetch(`/sha1/${resource.requestSha1}`);
+        if (resource.request.postData._sha1) {
-        const requestResource = await response.text();
+          const response = await fetch(`/sha1/${resource.request.postData}`);
-        setRequestBody(requestResource);
+          const requestResource = await response.text();
+          setRequestBody(requestResource);
+        } else {
+          setRequestBody(resource.request.postData.text);
+        }
       }
 
-      if (resource.responseSha1) {
+      if (resource.response.content._sha1) {
-        const useBase64 = resource.contentType.includes('image');
+        const useBase64 = resource.response.content.mimeType.includes('image');
-        const response = await fetch(`/sha1/${resource.responseSha1}`);
+        const response = await fetch(`/sha1/${resource.response.content._sha1}`);
         if (useBase64) {
           const blob = await response.blob();
           const reader = new FileReader();
@@ -58,7 +62,7 @@ export const NetworkResourceDetails: React.FunctionComponent<{
     };
 
     readResources();
-  }, [expanded, resource.responseSha1, resource.requestSha1, resource.contentType]);
+  }, [expanded, resource]);
 
   function formatBody(body: string | null, contentType: string): string {
     if (body === null)
@@ -93,33 +97,33 @@ export const NetworkResourceDetails: React.FunctionComponent<{
     return 'status-neutral';
   }
 
-  const requestContentTypeHeader = resource.requestHeaders.find(q => q.name === 'Content-Type');
+  const requestContentTypeHeader = resource.request.headers.find(q => q.name === 'Content-Type');
   const requestContentType = requestContentTypeHeader ? requestContentTypeHeader.value : '';
-  const resourceName = resource.url.substring(resource.url.lastIndexOf('/') + 1);
+  const resourceName = resource.request.url.substring(resource.request.url.lastIndexOf('/') + 1);
 
   return <div
     className={'network-request ' + (selected ? 'selected' : '')} onClick={() => setSelected(index)}>
     <Expandable expanded={expanded} setExpanded={setExpanded} style={{ width: '100%' }} title={
       <div className='network-request-title'>
-        <div className={'network-request-title-status ' + formatStatus(resource.status)}>{resource.status}</div>
+        <div className={'network-request-title-status ' + formatStatus(resource.response.status)}>{resource.response.status}</div>
-        <div className='network-request-title-method'>{resource.method}</div>
+        <div className='network-request-title-method'>{resource.request.method}</div>
         <div className='network-request-title-url'>{resourceName}</div>
-        <div className='network-request-title-content-type'>{resource.type}</div>
+        <div className='network-request-title-content-type'>{resource.response.content.mimeType}</div>
       </div>
     } body={
       <div className='network-request-details'>
         <div className='network-request-details-header'>URL</div>
-        <div className='network-request-details-url'>{resource.url}</div>
+        <div className='network-request-details-url'>{resource.request.url}</div>
         <div className='network-request-details-header'>Request Headers</div>
-        <div className='network-request-headers'>{resource.requestHeaders.map(pair => `${pair.name}: ${pair.value}`).join('\n')}</div>
+        <div className='network-request-headers'>{resource.request.headers.map(pair => `${pair.name}: ${pair.value}`).join('\n')}</div>
         <div className='network-request-details-header'>Response Headers</div>
-        <div className='network-request-headers'>{resource.responseHeaders.map(pair => `${pair.name}: ${pair.value}`).join('\n')}</div>
+        <div className='network-request-headers'>{resource.response.headers.map(pair => `${pair.name}: ${pair.value}`).join('\n')}</div>
-        {resource.requestSha1 ? <div className='network-request-details-header'>Request Body</div> : ''}
+        {resource.request.postData ? <div className='network-request-details-header'>Request Body</div> : ''}
-        {resource.requestSha1 ? <div className='network-request-body'>{formatBody(requestBody, requestContentType)}</div> : ''}
+        {resource.request.postData ? <div className='network-request-body'>{formatBody(requestBody, requestContentType)}</div> : ''}
         <div className='network-request-details-header'>Response Body</div>
-        {!resource.responseSha1 ? <div className='network-request-response-body'>Response body is not available for this request.</div> : ''}
+        {!resource.response.content._sha1 ? <div className='network-request-response-body'>Response body is not available for this request.</div> : ''}
         {responseBody !== null && responseBody.dataUrl ? <img src={responseBody.dataUrl} /> : ''}
-        {responseBody !== null && responseBody.text ? <div className='network-request-response-body'>{formatBody(responseBody.text, resource.contentType)}</div> : ''}
+        {responseBody !== null && responseBody.text ? <div className='network-request-response-body'>{formatBody(responseBody.text, resource.response.content.mimeType)}</div> : ''}
       </div>
     }/>
   </div>;

tests/snapshotter.spec.ts

@@ -146,7 +146,7 @@ it.describe('snapshots', () => {
     await page.evaluate(() => { (document.styleSheets[0].cssRules[0] as any).style.color = 'blue'; });
     const snapshot2 = await snapshotter.captureSnapshot(toImpl(page), 'snapshot1');
     const resource = snapshot2.resourceByUrl(`http://localhost:${server.PORT}/style.css`);
-    expect(snapshotter.resourceContent(resource.responseSha1).toString()).toBe('button { color: blue; }');
+    expect(snapshotter.resourceContent(resource.response.content._sha1).toString()).toBe('button { color: blue; }');
   });
 
   it('should capture iframe', async ({ page, server, toImpl, browserName, snapshotter, showSnapshot }) => {

tests/tracing.spec.ts

@@ -37,8 +37,8 @@ test('should collect trace with resources, but no js', async ({ context, page, s
   expect(events.find(e => e.metadata?.apiName === 'page.close')).toBeTruthy();
 
   expect(events.some(e => e.type === 'frame-snapshot')).toBeTruthy();
-  expect(events.some(e => e.type === 'resource-snapshot' && e.snapshot.url.endsWith('style.css'))).toBeTruthy();
+  expect(events.some(e => e.type === 'resource-snapshot' && e.snapshot.request.url.endsWith('style.css'))).toBeTruthy();
-  expect(events.some(e => e.type === 'resource-snapshot' && e.snapshot.url.endsWith('script.js'))).toBeFalsy();
+  expect(events.some(e => e.type === 'resource-snapshot' && e.snapshot.request.url.endsWith('script.js'))).toBeFalsy();
   expect(events.some(e => e.type === 'screencast-frame')).toBeTruthy();
 });
 
@@ -239,7 +239,7 @@ test('should reset and export', async ({ context, page, server }, testInfo) => {
   expect(trace1.events.find(e => e.metadata?.apiName === 'page.hover')).toBeFalsy();
   expect(trace1.events.find(e => e.metadata?.apiName === 'page.click' && e.metadata?.error?.error?.message === 'Action was interrupted')).toBeTruthy();
   expect(trace1.events.some(e => e.type === 'frame-snapshot')).toBeTruthy();
-  expect(trace1.events.some(e => e.type === 'resource-snapshot' && e.snapshot.url.endsWith('style.css'))).toBeTruthy();
+  expect(trace1.events.some(e => e.type === 'resource-snapshot' && e.snapshot.request.url.endsWith('style.css'))).toBeTruthy();
 
   const trace2 = await parseTrace(testInfo.outputPath('trace2.zip'));
   expect(trace2.events[0].type).toBe('context-options');